From 21cbf09c4aa609b5c61bed4bde514dc51d75623d Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Fri, 14 Jun 2024 15:56:31 -0400 Subject: [PATCH 01/11] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 774da8e..e9e7dfe 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,9 @@ https://github.com/ServiceNow/WorkArena/assets/1726818/ca26dfaf-2358-4418-855f-8 ### Dashboards -**Goal:** The agent must extract information from a dashboard. +**Goal:** The agent must answer a question that requires reading charts and (optionally) performing simple reasoning over them. +https://github.com/ServiceNow/WorkArena/assets/1726818/0023232c-081f-4be4-99bd-f60c766e6c3f ## Getting Started From 6bda558a90e0f024f9b855797e0c85163d57388f Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Fri, 14 Jun 2024 16:14:50 -0400 Subject: [PATCH 02/11] Update README.md --- README.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e9e7dfe..45a34de 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,9 @@ WorkArena is included in [BrowserGym](https://github.com/ServiceNow/BrowserGym), https://github.com/ServiceNow/WorkArena/assets/2374980/68640f09-7d6f-4eb1-b556-c294a6afef70 -## ⚠️ Pre-Release warning ⚠️ -Please note that the WorkArena benchmark is still undergoing minor bug fixes and updates, which may cause discrepancies with results reported in our latest arXiv preprint. We plan to release soon a stable version of WorkArena with enhanced stability, and a final version v1.0.0 with a new suite of tasks. - ## Benchmark Contents -At the moment, WorkArena includes `19,951` task instances drawn from `33` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point. +At the moment, WorkArena includes `19,912` unique instances drawn from `33` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point. ### Knowledge Bases @@ -99,6 +96,8 @@ Your installation is now complete! 🎉 Run this code to see WorkArena in action. +Note: the following example executes WorkArena's oracle (cheat) function to solve each task. To evaluate an agent, calls to `env.step()` must be used instead. + ```python import random @@ -119,22 +118,22 @@ for task in ALL_WORKARENA_TASKS: # Cheat functions use Playwright to automatically solve the task env.chat.add_message(role="assistant", msg="On it. Please wait...") - env.task.cheat(env.page, env.chat.messages) + cheat_messages = [] + env.task.cheat(env.page, messages) + + # Send cheat messages to chat + for cheat_msg in cheat_messages: + env.chat.add_message(role=cheat_msg["role"], msg=cheat_msg["message"]) # Post solution to chat - if "KnowledgeBaseSearchTask" in str(task): - answer = env.chat.messages[-1]["message"] - env.chat.add_message(role="assistant", msg=f"The answer is:") - env.chat.add_message(role="assistant", msg=answer) - else: - env.chat.add_message(role="assistant", msg="I'm done!") + env.chat.add_message(role="assistant", msg="I'm done!") # Validate the solution - reward, stop, info, message = env.task.validate(env.page, env.chat.messages) + reward, stop, message, info = env.task.validate(env.page, env.chat.messages) if reward == 1: env.chat.add_message(role="user", msg="Yes, that works. Thanks!") else: - env.chat.add_message(role="user", msg=f"No, that doesn't work. {message.get('message', '')}") + env.chat.add_message(role="user", msg=f"No, that doesn't work. {info.get('message', '')}") sleep(3) env.close() From d81c9f62c5d6f311c888d899fc72c97516e6bb03 Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Fri, 14 Jun 2024 16:30:37 -0400 Subject: [PATCH 03/11] Internal repo sync --- pyproject.toml | 1 + requirements.txt | 2 +- scripts/generate_knowledge_base.ipynb | 2831 +++++++++-------- scripts/make_human_eval_curriculum.py | 44 + scripts/wa_action_traces.py | 131 + src/browsergym/workarena/__init__.py | 113 +- src/browsergym/workarena/api/category.py | 74 + .../workarena/api/change_request.py | 87 + .../workarena/api/computer_asset.py | 90 + src/browsergym/workarena/api/cost_center.py | 19 + src/browsergym/workarena/api/expense_line.py | 89 + src/browsergym/workarena/api/incident.py | 45 + src/browsergym/workarena/api/knowledge.py | 29 + src/browsergym/workarena/api/problem.py | 90 + src/browsergym/workarena/api/report.py | 183 ++ .../workarena/api/requested_items.py | 63 + src/browsergym/workarena/api/user.py | 19 +- src/browsergym/workarena/api/utils.py | 50 +- src/browsergym/workarena/config.py | 22 +- .../forms/expected_incident_form_fields.json | 2 +- .../expected_request_item_form_fields.json | 1 + .../setup_files/knowledge/protocols.json | 46 + .../setup_files/knowledge/test.html | 1 + .../lists/expected_asset_list_columns.json | 26 +- .../expected_change_request_list_columns.json | 44 +- .../expected_expense_line_list_columns.json | 12 + .../lists/expected_hardware_list_columns.json | 43 +- .../lists/expected_incident_list_columns.json | 20 +- .../lists/expected_problem_list_columns.json | 12 + ...expected_requested_items_list_columns.json | 12 + ...expected_service_catalog_list_columns.json | 21 +- .../lists/expected_user_list_columns.json | 53 +- .../data_files/task_configs/all_menu.json | 2 +- .../dashboard_retrieval_minmax_task.json | 2 +- .../dashboard_retrieval_value_task.json | 2 +- ...filter_service_catalog_item_list_task.json | 2 +- .../task_configs/impersonation_users.json | 2 +- .../report_retrieval_minmax_task.json | 2 +- .../report_retrieval_value_task.json | 2 +- .../workarena/human_eval/console.js | 176 + src/browsergym/workarena/human_eval/tool.py | 366 +++ src/browsergym/workarena/install.py | 101 +- src/browsergym/workarena/tasks/base.py | 75 +- .../workarena/tasks/comp_building_block.py | 4 + .../workarena/tasks/compositional/__init__.py | 76 + .../workarena/tasks/compositional/base.py | 364 +++ .../tasks/compositional/dash_do_base.py | 1366 ++++++++ .../tasks/compositional/dash_do_catalog.py | 1127 +++++++ .../dash_do_catalog_infeasible.py | 2047 ++++++++++++ .../compositional/dash_do_create_incident.py | 403 +++ .../dash_do_create_incident_infeasible.py | 278 ++ .../compositional/dash_do_create_problem.py | 336 ++ .../dash_do_create_problem_infeasible.py | 235 ++ .../tasks/compositional/dash_do_filter.py | 1600 ++++++++++ .../compositional/dash_do_request_item.py | 1315 ++++++++ .../dash_do_request_item_infeasible.py | 693 ++++ .../tasks/compositional/delete_record.py | 341 ++ .../compositional/edit_knowledge_base.py | 457 +++ .../tasks/compositional/expense_management.py | 598 ++++ .../tasks/compositional/filter_and_do.py | 139 + .../compositional/find_and_order_item.py | 345 ++ .../manage_change_request_schedule.py | 1417 +++++++++ .../compositional/mark_duplicate_problems.py | 499 +++ .../maximize_investment_return.py | 1763 ++++++++++ .../tasks/compositional/navigate_and_do.py | 1151 +++++++ .../navigate_and_do_infeasible.py | 2100 ++++++++++++ .../tasks/compositional/offboard_user.py | 207 ++ .../tasks/compositional/onboard_user.py | 226 ++ .../tasks/compositional/update_task.py | 145 + .../tasks/compositional/utils/curriculum.py | 215 ++ .../compositional/utils/infeasible_configs.py | 151 + .../tasks/compositional/utils/knapsack.py | 192 ++ .../tasks/compositional/warranty_check.py | 227 ++ .../tasks/compositional/work_assignment.py | 804 +++++ .../tasks/compositional/workload_balancing.py | 396 +++ src/browsergym/workarena/tasks/dashboard.py | 196 +- src/browsergym/workarena/tasks/form.py | 1256 ++++++-- src/browsergym/workarena/tasks/knowledge.py | 241 +- src/browsergym/workarena/tasks/list.py | 621 +++- .../workarena/tasks/mark_duplicate_problem.py | 171 + src/browsergym/workarena/tasks/navigation.py | 68 +- .../tasks/scripts/extract_all_menu_items.py | 11 +- .../scripts/generate_dashboard_configs.py | 11 +- .../tasks/scripts/service_catalog.py | 3 +- .../workarena/tasks/scripts/validate.py | 10 +- .../workarena/tasks/send_chat_message.py | 90 + .../workarena/tasks/service_catalog.py | 120 +- src/browsergym/workarena/tasks/utils/form.py | 5 +- .../workarena/tasks/utils/private_tasks.py | 63 + src/browsergym/workarena/tasks/utils/utils.py | 13 + tests/test_api.py | 1 + tests/test_compositional.py | 175 + tests/test_compositional_utils.py | 92 + tests/test_random_config_generation.py | 47 +- tests/test_task_from_config.py | 40 +- tests/test_task_general.py | 13 +- 96 files changed, 27408 insertions(+), 2063 deletions(-) create mode 100644 scripts/make_human_eval_curriculum.py create mode 100644 scripts/wa_action_traces.py create mode 100644 src/browsergym/workarena/api/category.py create mode 100644 src/browsergym/workarena/api/change_request.py create mode 100644 src/browsergym/workarena/api/computer_asset.py create mode 100644 src/browsergym/workarena/api/cost_center.py create mode 100644 src/browsergym/workarena/api/expense_line.py create mode 100644 src/browsergym/workarena/api/incident.py create mode 100644 src/browsergym/workarena/api/knowledge.py create mode 100644 src/browsergym/workarena/api/problem.py create mode 100644 src/browsergym/workarena/api/report.py create mode 100644 src/browsergym/workarena/api/requested_items.py create mode 100644 src/browsergym/workarena/data_files/setup_files/forms/expected_request_item_form_fields.json create mode 100644 src/browsergym/workarena/data_files/setup_files/knowledge/protocols.json create mode 100644 src/browsergym/workarena/data_files/setup_files/knowledge/test.html create mode 100644 src/browsergym/workarena/data_files/setup_files/lists/expected_expense_line_list_columns.json create mode 100644 src/browsergym/workarena/data_files/setup_files/lists/expected_problem_list_columns.json create mode 100644 src/browsergym/workarena/data_files/setup_files/lists/expected_requested_items_list_columns.json create mode 100644 src/browsergym/workarena/human_eval/console.js create mode 100644 src/browsergym/workarena/human_eval/tool.py create mode 100644 src/browsergym/workarena/tasks/comp_building_block.py create mode 100644 src/browsergym/workarena/tasks/compositional/__init__.py create mode 100644 src/browsergym/workarena/tasks/compositional/base.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_base.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_catalog.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_catalog_infeasible.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_create_incident.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_create_incident_infeasible.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_create_problem.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_create_problem_infeasible.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_filter.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_request_item.py create mode 100644 src/browsergym/workarena/tasks/compositional/dash_do_request_item_infeasible.py create mode 100644 src/browsergym/workarena/tasks/compositional/delete_record.py create mode 100644 src/browsergym/workarena/tasks/compositional/edit_knowledge_base.py create mode 100644 src/browsergym/workarena/tasks/compositional/expense_management.py create mode 100644 src/browsergym/workarena/tasks/compositional/filter_and_do.py create mode 100644 src/browsergym/workarena/tasks/compositional/find_and_order_item.py create mode 100644 src/browsergym/workarena/tasks/compositional/manage_change_request_schedule.py create mode 100644 src/browsergym/workarena/tasks/compositional/mark_duplicate_problems.py create mode 100644 src/browsergym/workarena/tasks/compositional/maximize_investment_return.py create mode 100644 src/browsergym/workarena/tasks/compositional/navigate_and_do.py create mode 100644 src/browsergym/workarena/tasks/compositional/navigate_and_do_infeasible.py create mode 100644 src/browsergym/workarena/tasks/compositional/offboard_user.py create mode 100644 src/browsergym/workarena/tasks/compositional/onboard_user.py create mode 100644 src/browsergym/workarena/tasks/compositional/update_task.py create mode 100644 src/browsergym/workarena/tasks/compositional/utils/curriculum.py create mode 100644 src/browsergym/workarena/tasks/compositional/utils/infeasible_configs.py create mode 100644 src/browsergym/workarena/tasks/compositional/utils/knapsack.py create mode 100644 src/browsergym/workarena/tasks/compositional/warranty_check.py create mode 100644 src/browsergym/workarena/tasks/compositional/work_assignment.py create mode 100644 src/browsergym/workarena/tasks/compositional/workload_balancing.py create mode 100644 src/browsergym/workarena/tasks/mark_duplicate_problem.py create mode 100644 src/browsergym/workarena/tasks/send_chat_message.py create mode 100644 src/browsergym/workarena/tasks/utils/private_tasks.py create mode 100644 tests/test_compositional.py create mode 100644 tests/test_compositional_utils.py diff --git a/pyproject.toml b/pyproject.toml index ae4a017..95781ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ homepage = "https://github.com/ServiceNow/WorkArena" [project.scripts] workarena-install = "browsergym.workarena.install:main" +workarena-human-eval = "browsergym.workarena.human_eval.tool:main" [tool.hatch.version] path = "src/browsergym/workarena/__init__.py" diff --git a/requirements.txt b/requirements.txt index 1cef95a..5b9b49e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ browsergym-core>=0.2 english-words>=2.0.1 -faker>=24.11.0 +Faker>=24.8.0 numpy>=1.14 requests>=2.31 tenacity>=8.2.3 # only used in cheat() -> move to tests? diff --git a/scripts/generate_knowledge_base.ipynb b/scripts/generate_knowledge_base.ipynb index ac76543..cb7fc0c 100644 --- a/scripts/generate_knowledge_base.ipynb +++ b/scripts/generate_knowledge_base.ipynb @@ -1,1374 +1,1499 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "R514MKNqG27Y" - }, - "source": [ - "# WorkArena Knowledge Base\n", - "Author: Alexandre Drouin (alexandre.drouin@servicenow.com)\n", - "\n", - "\n", - "This notebook contains code to generate:\n", - "* Knowledge base articles given a list of facts\n", - "* Questions to query those articles for given facts\n", - "* Multiple alternative wordings of the expected answers" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "R514MKNqG27Y" + }, + "source": [ + "# WorkArena Knowledge Base\n", + "Author: Alexandre Drouin (alexandre.drouin@servicenow.com)\n", + "\n", + "\n", + "This notebook contains code to generate:\n", + "* Knowledge base articles given a list of facts\n", + "* Questions to query those articles for given facts\n", + "* Multiple alternative wordings of the expected answers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "R3kofpiH7Bl2", + "outputId": "d3d48e02-2ded-441f-b632-9fb43976b64d" + }, + "outputs": [], + "source": [ + "!pip install openai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PAK3oqCG61PZ" + }, + "outputs": [], + "source": [ + "import json\n", + "import openai\n", + "import os\n", + "\n", + "\n", + "client = openai.OpenAI(api_key=os.environ[\"OPENAI_API_KEY\"])\n", + "\n", + "\n", + "def chat(messages, model=\"gpt-4-1106-preview\"):\n", + " return (\n", + " client.chat.completions.create(model=model, messages=messages)\n", + " .choices[0]\n", + " .message\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OGT5ZjrK7HVs" + }, + "outputs": [], + "source": [ + "def generate_article(item, value, all_facts, n_retries=5):\n", + " prompt = f\"\"\"\n", + "\n", + "You are in charge of writing knowledge-base articles to document important company and workplace information.\n", + "Your articles will be used as reference by the employees of the company.\n", + "\n", + "\n", + "Here is a list of facts in the knowledge base:\n", + "\n", + "{all_facts}\n", + "\n", + "\n", + "You need to write an article about this specific fact:\n", + "\n", + "The {item} is {value}.\n", + "\n", + "\n", + "\n", + "* Generate a knowledge-base article that contains the as-is. Do not modify or add to its text.\n", + "* Hide the fact inside a bunch of other related information, but do not source the related information from . Make stuff up.\n", + "* Make sure that nothing you write contradicts \n", + "* You must use HTML format. Generate only the tag.\n", + "* Don't include information about the knowledge base itself or headers like \\\"welcome to our knowledge base\\\".\n", + "\n", + "\n", + "\"\"\"\n", + " messages = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a ServiceNow system administrator in charge of writing knowledge base articles to help employees in your organization.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": prompt},\n", + " ]\n", + "\n", + " print(\"... attempting to produce article\")\n", + " for retry in range(n_retries):\n", + " try:\n", + " print(f\"... try {retry}\")\n", + " messages.append({\"role\": \"assistant\", \"content\": chat(messages).content})\n", + " article = (\n", + " messages[-1][\"content\"]\n", + " .replace(\"\", \"\")\n", + " .replace(\"\", \"\")\n", + " .replace(\"```HTML\", \"\")\n", + " .replace(\"```\", \"\")\n", + " .replace(\"\", \"\")\n", + " .replace(\"\", \"\")\n", + " .strip()\n", + " )\n", + "\n", + " # Validate that the fact is included without modification\n", + " assert (\n", + " f\"the {item} is {value}\".lower() in article.lower()\n", + " ), f'Error: Could not find the string \"The {item} is {value}\" in this article.'\n", + " print(\"... valid article found.\")\n", + " break\n", + " except AssertionError as e:\n", + " messages.append(\n", + " {\"role\": \"user\", \"content\": str(e) + \"\\nDon't apologize and try again.\"}\n", + " )\n", + "\n", + " if messages[-1][\"role\"] == \"user\":\n", + " raise RuntimeError(\n", + " f\"Failed to produce a valid article after {n_retries} retries.\"\n", + " )\n", + "\n", + " return article" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "BuAPHUc18EDy", + "outputId": "f58c7b3b-cdef-4c04-a946-be6f817cca80" + }, + "outputs": [], + "source": [ + "facts = [\n", + " {\"item\": \"password to conference room A-561\", \"value\": \"roo918k\"},\n", + " {\"item\": \"address of office #456\", \"value\": \"42, Pizza street, New York, USA\"},\n", + " {\"item\": \"number of employees in department X\", \"value\": \"75\"},\n", + " {\"item\": \"CEO's name\", \"value\": \"Alex Johnson\"},\n", + " {\"item\": \"year of company establishment\", \"value\": \"1998\"},\n", + " {\"item\": \"Wi-Fi network name in office #456\", \"value\": \"Office456_WiFi\"},\n", + " {\"item\": \"Wi-Fi password for office #456\", \"value\": \"456SecureNet!\"},\n", + " {\"item\": \"average annual revenue\", \"value\": \"$50 million\"},\n", + " {\"item\": \"number of branches worldwide\", \"value\": \"18\"},\n", + " {\"item\": \"name of the head of HR department\", \"value\": \"Samantha Green\"},\n", + " {\"item\": \"brand of coffee machine in kitchen #3\", \"value\": \"Delonghi Magnifica\"},\n", + " {\"item\": \"company's stock ticker symbol\", \"value\": \"COMPX\"},\n", + " {\"item\": \"company's main product\", \"value\": \"Advanced Analytics Software\"},\n", + " {\"item\": \"color of the carpet in conference room A-561\", \"value\": \"Navy Blue\"},\n", + " {\"item\": \"annual budget for marketing department\", \"value\": \"$2 million\"},\n", + " {\"item\": \"capacity of conference room B-762\", \"value\": \"30 people\"},\n", + " {\"item\": \"number of floors in the main office building\", \"value\": \"10\"},\n", + " {\"item\": \"type of plants in the lobby of office #456\", \"value\": \"Ficus Lyrata\"},\n", + " {\n", + " \"item\": \"catering service provider for office events\",\n", + " \"value\": \"Gourmet Caterers\",\n", + " },\n", + " {\"item\": \"software used for payroll management\", \"value\": \"QuickBooks Payroll\"},\n", + " {\"item\": \"number of parking spaces in office #456 parking lot\", \"value\": \"150\"},\n", + " {\"item\": \"company's first product\", \"value\": \"Data Analysis Toolkit\"},\n", + " {\"item\": \"brand of computers used in IT department\", \"value\": \"Lenovo ThinkPad\"},\n", + " {\"item\": \"annual CSR budget\", \"value\": \"$500,000\"},\n", + " {\"item\": \"make of company's official car\", \"value\": \"Tesla Model X\"},\n", + " {\"item\": \"name of the company's auditor\", \"value\": \"PriceWaterhouseCoopers\"},\n", + " {\"item\": \"number of customer service representatives\", \"value\": \"40\"},\n", + " {\n", + " \"item\": \"type of air conditioning system in office #456\",\n", + " \"value\": \"Centralized HVAC\",\n", + " },\n", + " {\"item\": \"name of the cafeteria manager in office #456\", \"value\": \"Miguel Torres\"},\n", + " {\"item\": \"duration of lunch break in office #456\", \"value\": \"1 hour\"},\n", + " {\"item\": \"official language for company communication\", \"value\": \"English\"},\n", + " {\"item\": \"number of conference rooms in office #456\", \"value\": \"8\"},\n", + " {\"item\": \"average time for IT support response\", \"value\": \"15 minutes\"},\n", + " {\"item\": \"brand of printers in office #456\", \"value\": \"HP LaserJet Pro\"},\n", + " {\"item\": \"capacity of the largest meeting room\", \"value\": \"50 people\"},\n", + " {\"item\": \"number of patents held by the company\", \"value\": \"35\"},\n", + " {\"item\": \"company's main competitor\", \"value\": \"TechRivals Inc.\"},\n", + " {\"item\": \"company's slogan\", \"value\": \"Innovating the Future\"},\n", + " {\"item\": \"number of countries the company operates in\", \"value\": \"12\"},\n", + " {\n", + " \"item\": \"type of security system at office #456\",\n", + " \"value\": \"Biometric Access Control\",\n", + " },\n", + " {\"item\": \"name of the employee of the month\", \"value\": \"Elena Rodriguez\"},\n", + " {\"item\": \"brand of the photocopier in office #456\", \"value\": \"Canon ImageRunner\"},\n", + " {\"item\": \"type of coffee provided in the kitchen\", \"value\": \"Arabica Beans\"},\n", + " {\"item\": \"annual IT budget\", \"value\": \"$1.5 million\"},\n", + " {\"item\": \"name of the legal counsel firm\", \"value\": \"Baker & McKenzie\"},\n", + " {\"item\": \"average employee satisfaction score\", \"value\": \"8.5/10\"},\n", + " {\"item\": \"number of departments in the company\", \"value\": \"12\"},\n", + " {\"item\": \"company's largest client\", \"value\": \"GlobalTech Industries\"},\n", + " {\n", + " \"item\": \"type of gym equipment in fitness center\",\n", + " \"value\": \"Life Fitness machines\",\n", + " },\n", + " {\"item\": \"average number of yearly hires\", \"value\": \"100\"},\n", + " {\"item\": \"brand of air purifiers used in the office\", \"value\": \"Dyson Pure Cool\"},\n", + " {\n", + " \"item\": \"name of the employee health insurance provider\",\n", + " \"value\": \"BlueCross BlueShield\",\n", + " },\n", + " {\"item\": \"number of projects completed last year\", \"value\": \"22\"},\n", + " {\"item\": \"company's main export market\", \"value\": \"European Union\"},\n", + " {\n", + " \"item\": \"name of the office cleaning service provider\",\n", + " \"value\": \"CleanSweep Inc.\",\n", + " },\n", + " {\"item\": \"type of video conferencing software used\", \"value\": \"Zoom\"},\n", + " {\"item\": \"color of the company logo\", \"value\": \"Royal Blue and Silver\"},\n", + " {\"item\": \"average client retention rate\", \"value\": \"85%\"},\n", + " {\"item\": \"name of the company's largest shareholder\", \"value\": \"David Thompson\"},\n", + " {\"item\": \"total square footage of office #456\", \"value\": \"25,000 square feet\"},\n", + " {\n", + " \"item\": \"type of backup power system in office #456\",\n", + " \"value\": \"Diesel Generators\",\n", + " },\n", + " {\"item\": \"average yearly expenditure on office supplies\", \"value\": \"$80,000\"},\n", + " {\"item\": \"name of the cafeteria food supplier\", \"value\": \"FreshFoods Ltd.\"},\n", + " {\n", + " \"item\": \"type of fire safety system in office #456\",\n", + " \"value\": \"Automatic Sprinkler System\",\n", + " },\n", + " {\"item\": \"company's primary industry\", \"value\": \"Technology\"},\n", + " {\"item\": \"number of annual company-wide meetings\", \"value\": \"4\"},\n", + " {\"item\": \"brand of smartphones provided to employees\", \"value\": \"Samsung Galaxy\"},\n", + " {\"item\": \"name of the software used for project management\", \"value\": \"Trello\"},\n", + " {\"item\": \"average number of business trips per employee annually\", \"value\": \"3\"},\n", + " {\"item\": \"number of IT support staff\", \"value\": \"15\"},\n", + " {\n", + " \"item\": \"company's primary social media platform for marketing\",\n", + " \"value\": \"LinkedIn\",\n", + " },\n", + " {\"item\": \"number of active contracts\", \"value\": \"40\"},\n", + " {\"item\": \"annual expenditure on R&D\", \"value\": \"$3 million\"},\n", + " {\n", + " \"item\": \"type of chairs used in conference room A-561\",\n", + " \"value\": \"Ergonomic Office Chairs\",\n", + " },\n", + " {\n", + " \"item\": \"name of the most used software by the design team\",\n", + " \"value\": \"Adobe Creative Suite\",\n", + " },\n", + " {\"item\": \"average delivery time for company products\", \"value\": \"5 business days\"},\n", + " {\"item\": \"type of lighting in office #456\", \"value\": \"LED Lights\"},\n", + " {\"item\": \"average monthly electricity bill for office #456\", \"value\": \"$10,000\"},\n", + " {\"item\": \"company's largest expense category\", \"value\": \"Employee Salaries\"},\n", + " {\"item\": \"brand of the refrigerator in the kitchen\", \"value\": \"LG InstaView\"},\n", + " {\n", + " \"item\": \"type of health and safety training provided\",\n", + " \"value\": \"First Aid and Fire Safety\",\n", + " },\n", + " {\n", + " \"item\": \"name of the corporate social responsibility program\",\n", + " \"value\": \"TechForGood\",\n", + " },\n", + " {\"item\": \"annual water consumption in office #456\", \"value\": \"50,000 gallons\"},\n", + " {\"item\": \"company's policy on remote work\", \"value\": \"Hybrid Model\"},\n", + " {\"item\": \"number of customer complaints last year\", \"value\": \"120\"},\n", + " {\"item\": \"name of the most popular product\", \"value\": \"Smart Analytics Pro\"},\n", + " {\n", + " \"item\": \"type of snacks available in the kitchen\",\n", + " \"value\": \"Healthy and Organic Snacks\",\n", + " },\n", + " {\"item\": \"average duration of employee tenure\", \"value\": \"5 years\"},\n", + " {\n", + " \"item\": \"brand of the security cameras in office #456\",\n", + " \"value\": \"Axis Communications\",\n", + " },\n", + " {\"item\": \"type of heating system in office #456\", \"value\": \"Forced Air Heating\"},\n", + " {\n", + " \"item\": \"name of the internet service provider for office #456\",\n", + " \"value\": \"Comcast Xfinity\",\n", + " },\n", + " {\n", + " \"item\": \"company's policy on environmental sustainability\",\n", + " \"value\": \"Zero-Waste Initiatives\",\n", + " },\n", + " {\n", + " \"item\": \"name of the annual team-building retreat location\",\n", + " \"value\": \"Lakeview Resort\",\n", + " },\n", + " {\n", + " \"item\": \"type of vending machines in office #456\",\n", + " \"value\": \"Cashless Payment Vending\",\n", + " },\n", + " {\"item\": \"number of team leaders in the company\", \"value\": \"35\"},\n", + " {\"item\": \"annual spending on employee training programs\", \"value\": \"$250,000\"},\n", + " {\"item\": \"CEO's direct phone line\", \"value\": \"+1-555-1010-2020\"},\n", + " {\"item\": \"Wi-Fi network name in the lobby\", \"value\": \"CorporateGuest123\"},\n", + " {\"item\": \"last fire drill date at main building\", \"value\": \"2023-06-15\"},\n", + " {\"item\": \"number of employees in marketing department\", \"value\": \"35\"},\n", + "]\n", + "\n", + "print(\"Number of facts:\", len(facts))\n", + "\n", + "for i, f in enumerate(facts):\n", + " f[\"article\"] = generate_article(f[\"item\"], f[\"value\"], all_facts=facts)\n", + " print(\"... Article\", i + 1, f[\"item\"], f[\"value\"])\n", + " print(f[\"article\"])\n", + " print(\"\\n\" * 2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 262 + }, + "id": "tDTelvGXWp-P", + "outputId": "ddb6f999-f127-4e25-bb9f-56037a4fb7ed" + }, + "outputs": [], + "source": [ + "for f in facts:\n", + " f[\"article\"] = (\n", + " f[\"article\"]\n", + " .replace(\"```HTML\", \"\")\n", + " .replace(\"```\", \"\")\n", + " .replace(\"\", \"\")\n", + " .replace(\"\", \"\")\n", + " .strip()\n", + " )\n", + "\n", + "json.dump(facts, open(\"knowledge_base.json\", \"w\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ctj7CPtAon3w" + }, + "source": [ + "# Generate questions and answers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QrPIz4CZ3ier", + "outputId": "ceb47583-81ea-4f21-9fa5-dff4045d16c9" + }, + "outputs": [], + "source": [ + "import concurrent.futures\n", + "import re\n", + "\n", + "from collections import Counter\n", + "from tqdm.notebook import tqdm\n", + "\n", + "\n", + "# Reload the generated KB\n", + "kb = json.load(open(\"knowledge_base.json\", \"r\"))\n", + "print(\"Loaded\", len(kb), \"articles\")\n", + "\n", + "\n", + "# Clear all questions\n", + "for kb_i in kb:\n", + " if \"questions\" in kb_i:\n", + " kb_i[\"questions\"] = []\n", + " kb_i[\"alternative_answers\"] = []" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ee_KMC2o3gDD" + }, + "source": [ + "## Generate several variants of the question" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "e77eceb6a8624f66ac12cc3df788b37e", + "ddb341b6b65a4ab19da7c1811a706be7", + "b2de9413f61b4be7947f9bc0d3f12f41", + "dd4959d2ace04d5eb2a8c3274a069647", + "be1c8e816d0743eaa5c49d26e0050e94", + "6bbe2aa489d6427daca0d0ac04a51f6e", + "1cfe99f3f2af48e1800ad53f911de327", + "d7c95fe313c546a6af713d3cb0e004d6", + "a1221b65b0644cd794dc477d0492979f", + "511c96890e1a4989bae8390ca627ac37", + "0e044725d805479b9038d0552f277a7e" + ] + }, + "id": "NLERyQk-Xge5", + "outputId": "0fb2d008-ac09-4cdf-f63d-cc623df09b63" + }, + "outputs": [], + "source": [ + "def get_most_frequent_items(my_list):\n", + " # Count the frequency of each item in the list\n", + " frequency = Counter(my_list)\n", + "\n", + " # Find the maximum frequency\n", + " max_frequency = max(frequency.values())\n", + "\n", + " # Get all items with the maximum frequency\n", + " most_frequent = [\n", + " item for item, count in frequency.items() if count == max_frequency\n", + " ]\n", + "\n", + " return most_frequent\n", + "\n", + "\n", + "def generate_question(article, item, value, n_questions=10, n_retries=5):\n", + " prompt = f\"\"\"\n", + " Here is an article taken from a company knowledge base.\n", + "\n", + "
\n", + " {article}\n", + "
\n", + "\n", + " This article contains the following fact:\n", + " \n", + " The {item} is {value}.\n", + " \n", + "\n", + " Here is a question about this fact to which the answer is \\\"{value}\\\":\n", + " \n", + " What is the {item}?\n", + " \n", + "\n", + " Produce {n_questions} rephrasings of this question.\n", + "\n", + " \n", + " * Make sure that your questions are precise and unambiguous.\n", + " * It must be clear that the questions are asking about \\\"{item}\\\" and\n", + " their answer must still be exactly \\\"{value}\\\".\n", + " * Make sure that you provide clear and specific instructions on the expected\n", + " format for the answer (e.g., Day, Month, Year).\n", + " * You cannot, in any circumstances, reveal information from the answer in the question or instructions.\n", + " * Make sure they are questions that end with a question mark.\n", + " * Make sure that your questions do not mention the article (e.g., \\\"in the article\\\").\n", + " * Answer with one per line and do not number them.\n", + " \n", + "\n", + " \n", + " Suppose the question is \\\"On which day was the company founded?\\\" and the answer is \\\"January 26, 2024\\\",\n", + " then a good rephrasing would be \\\"When was the company founded? Answer with Month Day, Year\\\".\n", + " \n", + "\n", + " \n", + " Suppose the question is \\\"What kind of fridge do we have in the cafeteria?\\\" and the answer is \\\"LG X456\\\",\n", + " then a good rephrasing would be \\\"Which type of fridge is in the cafeteria? Answer with Brand followed by Model name\\\".\n", + " \n", + "\n", + " \n", + " Suppose the question is \\\"Where is the company headquarter located?\\\" and the answer is \\\"123, Banana Street, Montreal, Canada\\\",\n", + " then a good rephrasing would be \\\"What's the address of the company headquarter? Answer with Number, Street, City, Country\\\".\n", + " \n", + "\n", + " \n", + " Suppose the question is \\\"What were the total sales of the company in 2023?\\\" and the answer is \\\"150B$\\\",\n", + " then a good rephrasing would be \\\"What do the company sales for 2023 sum up to? Answer with NumberB$, where B is billions\\\".\n", + " \n", + " \"\"\"\n", + " good_questions = set([])\n", + " messages = [{\"role\": \"user\", \"content\": prompt}]\n", + " while len(good_questions) < n_questions:\n", + " try:\n", + " # Generate questions\n", + " questions = chat(messages).content\n", + "\n", + " # Parse output\n", + " # ... check the article is not mentioned\n", + " assert (\n", + " \"article\" not in questions.lower()\n", + " ), \"You are not allowed to mention the article in your rephrasing of the questions.\"\n", + " # ... check the number of actual questions included\n", + " assert (\n", + " questions.count(\"?\") == n_questions\n", + " ), f\"I couldn't find {n_questions} question marks in the output. Make sure all questions end with ?\"\n", + " # ... heuristic to detect numbered questions\n", + " assert not all(\n", + " f\"{x}.\" in questions for x in range(4)\n", + " ), \"The questions appear to be numbered, but they should not.\"\n", + " questions = [q.strip() for q in questions.split(\"\\n\") if q != \"\"]\n", + " # ... check one question per line\n", + " assert (\n", + " len(questions) == n_questions\n", + " ), f\"Your answer is not {n_questions} lines long. Make sure it contains {n_questions} questions and one per line.\"\n", + " # ... check each question has instructions\n", + " assert all(\n", + " len(q.split(\"?\")) > 1 and len(\"\".join(q.split(\"?\")[1:]).strip()) > 5\n", + " for q in questions\n", + " ), f'Make sure you provide clear formatting instructions for each question (e.g., \"Question? Answer with\").'\n", + " # ... check that answer is not mentioned\n", + " for q in questions:\n", + " error = \"Do not include the answer in the questions/instructions!\"\n", + " # ... exact value is not in the question\n", + " assert value.lower() not in q.lower(), error\n", + "\n", + " # Validate questions\n", + " bad_questions = []\n", + " bad_answers = []\n", + " for q in questions:\n", + " print(\"... testing:\", q, end=\" \")\n", + " success, answers = is_question_answerable(article, q, value)\n", + " if not success:\n", + " bad_questions.append(q)\n", + " bad_answers.append(answers)\n", + " print(\"FAIL\")\n", + " print(\"... preemptively stopping to give feedback\")\n", + " break\n", + " else:\n", + " good_questions.add(q)\n", + " print(\"PASS\")\n", + "\n", + " # Give feedback and retry\n", + " if len(bad_questions) > 0:\n", + " feedback = \"\"\n", + " for q, a in zip(bad_questions, bad_answers):\n", + " feedback += \"\\n\"\n", + " feedback += \" \\n\"\n", + " feedback += \" \" + q + \"\\n\"\n", + " feedback += \" \\n\"\n", + " feedback += \" \\n\"\n", + " feedback += \"\\n\".join(\" \" + x for x in a) + \"\\n\"\n", + " feedback += \" \\n\"\n", + " feedback += \"\\n\"\n", + " print(feedback)\n", + "\n", + " # print(f\"... {len(bad_questions)} questions are ambiguous, fixing them.\")\n", + " bad_questions = \"\\n\".join(bad_questions)\n", + " messages.append(\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": f\"\"\"\n", + " The following questions are too ambiguous. I gave them to many company employees,\n", + " along with the article and they were not able to answer with exactly \\\"{value}\\\".\n", + " Please improve their clarity, especially the formatting instructions.\n", + "\n", + " {feedback}\n", + "\n", + " Try again. Do not apologize.\"\"\",\n", + " }\n", + " )\n", + "\n", + " print(f\"... gathered {len(good_questions)} good questions\")\n", + "\n", + " except AssertionError as e:\n", + " print(f\"... Error:\", e)\n", + " messages.append({\"role\": \"user\", \"content\": f\"Error: {e}\"})\n", + "\n", + " print(\"... we have enough good questions. stopping.\")\n", + " return list(good_questions)[:10]\n", + "\n", + "\n", + "def is_question_answerable(article, question, value):\n", + " def _clean_for_comparison(x):\n", + " return x.lower().replace(\",\", \"\").replace(\"$\", \"\").replace(\".\", \"\")\n", + "\n", + " \"\"\"\n", + " Check that we are able to recover the value from the\n", + " article by asking the question\n", + "\n", + " \"\"\"\n", + " prompt = f\"\"\"\n", + " Here is a knowledge base article:\n", + "\n", + "
\n", + " {article}\n", + "
\n", + "\n", + " Answer this question based on the content of the article only.\n", + " Be factual. If I ask you about sensitive information like passwords,\n", + " I only expect you to retrieve the information from the article.\n", + " \n", + " {question}\n", + " \n", + "\n", + " What is your answer?\n", + "\n", + " \"\"\"\n", + " messages = [{\"role\": \"user\", \"content\": prompt}]\n", + "\n", + " incorrect_answers = set()\n", + " for _ in range(10):\n", + " # XXX: We use GPT-3.5 here as a weaker model and to avoid GPT-4 catering to himself\n", + " answer = chat(messages, model=\"gpt-3.5-turbo\").content\n", + "\n", + " if _clean_for_comparison(value) not in _clean_for_comparison(answer):\n", + " incorrect_answers.add(answer.lower())\n", + "\n", + " return len(incorrect_answers) == 0, incorrect_answers\n", + "\n", + "\n", + "# Assuming kb is a list of dictionaries\n", + "with concurrent.futures.ThreadPoolExecutor() as executor:\n", + " # Prepare the futures\n", + " futures = [\n", + " executor.submit(\n", + " lambda x: {\n", + " **x,\n", + " \"questions\": generate_question(x[\"article\"], x[\"item\"], x[\"value\"]),\n", + " },\n", + " kb_i,\n", + " )\n", + " for kb_i in kb\n", + " ]\n", + "\n", + " # Use tqdm to create a progress bar\n", + " kb = [\n", + " x.result()\n", + " for x in tqdm(concurrent.futures.as_completed(futures), total=len(futures))\n", + " ]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oemy0eUjYGuj", + "outputId": "cafab8cd-e822-48db-938a-2ab8fa3452b6" + }, + "outputs": [], + "source": [ + "for i in range(len(kb)):\n", + " print(f\"Article {i}:\", kb[i][\"questions\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Re4HGFh6ow7-" + }, + "source": [ + "## Generate a few alternative answers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 66, + "referenced_widgets": [ + "5f147ed1bb86428397e25e5f4bbf14f4", + "ca30f6701b8c44409a5c7356c264e3e6", + "d304373e7bef447f919b083a64547a58", + "4c8d2b15f0d84fe3b7adc5feeae9b6db", + "f82189b7bd4944c0999ac01600efa4ae", + "e69716214e49423898fdc173108cadbe", + "483996fae22d4d2e89f83c05c7db6e76", + "fdd79cc467e44d849d31b43b416bcb20", + "eba164765ad544c38a7679d494343551", + "571e7187a94d4860915f70f85654573b", + "16997d3f7a414ec79bcca88298fac743" + ] + }, + "id": "Dxjc0HQOo25w", + "outputId": "fdbd7723-1dda-4eda-f0ee-281fc051d04c" + }, + "outputs": [], + "source": [ + "def alternative_answers(article, item, value, questions, n_answers=10):\n", + " questions = \"\\n\".join(questions)\n", + "\n", + " prompt = f\"\"\"\n", + " Here is a set of questions:\n", + " \n", + " {questions}\n", + " \n", + "\n", + " The exact answer to all of these questions is:\n", + " \n", + " {value}\n", + " \n", + "\n", + " Give me {n_answers} other ways to spell out the answer.\n", + " Don't add context words around it or anything, just reformulate it.\n", + "\n", + " \n", + " Initial value: 5.5/10\n", + " Reformulated: 5.5 out of 10\n", + " Reformulated: 55%\n", + " Reformulated: fifty-five percent\n", + " \n", + "\n", + " \n", + " Initial value: Tesla Model X\n", + " Reformulated: Model X by Tesla\n", + " Reformulated: Tesla's Model X\n", + " \n", + "\n", + " \n", + " Initial value: 150$\n", + " Reformulated: one hundred fifty dollars\n", + " Reformulated: 150.00$\n", + " Reformulated: $150\n", + " \n", + "\n", + " Answer with one per line. Don't number them.\n", + "\n", + " \"\"\"\n", + " messages = [{\"role\": \"user\", \"content\": prompt}]\n", + "\n", + " while True:\n", + " try:\n", + " answers = chat(messages).content\n", + "\n", + " # Validation\n", + " # ... heuristic to detect numbered questions\n", + " assert not all(\n", + " f\"{x}.\" in answers for x in range(4)\n", + " ), \"The answers appear to be numbered, but they should not.\"\n", + " answers = [a.strip() for a in answers.split(\"\\n\") if a != \"\"]\n", + " # ... check one question per line\n", + " assert (\n", + " len(answers) == n_answers\n", + " ), f\"Your response is not {answers} lines long. Make sure it contains {answers} answers and one per line.\"\n", + " assert (\n", + " len(set(answers)) == n_answers\n", + " ), f\"You provided duplicate values. There were only {len(set(answers))} unique values.\"\n", + " break\n", + " except AssertionError as e:\n", + " print(f\"... Error:\", e)\n", + " messages.append({\"role\": \"user\", \"content\": f\"Error: {e}\"})\n", + "\n", + " return answers\n", + "\n", + "\n", + "for kb_i in tqdm(kb, total=len(kb)):\n", + " kb_i[\"alternative_answers\"] = alternative_answers(\n", + " kb_i[\"article\"], kb_i[\"item\"], kb_i[\"value\"], kb_i[\"questions\"]\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CFi8jwUhFUDP", + "outputId": "08a3c5c4-5da9-4fc4-a443-0e2ac1891f7e" + }, + "outputs": [], + "source": [ + "for kb_i in kb:\n", + " print(kb_i[\"alternative_answers\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i-CbZL-do1fX" + }, + "source": [ + "## Save it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Uww5Ut2rfR4f" + }, + "outputs": [], + "source": [ + "json.dump(kb, open(\"knowledge_base.json\", \"w\"))" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "0e044725d805479b9038d0552f277a7e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "R3kofpiH7Bl2", - "outputId": "d3d48e02-2ded-441f-b632-9fb43976b64d" - }, - "outputs": [], - "source": [ - "!pip install openai" - ] + "16997d3f7a414ec79bcca88298fac743": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "PAK3oqCG61PZ" - }, - "outputs": [], - "source": [ - "import json\n", - "import openai\n", - "import os\n", - "\n", - "\n", - "client = openai.OpenAI(api_key=os.environ[\"OPENAI_API_KEY\"])\n", - "\n", - "def chat(messages, model=\"gpt-4-1106-preview\"):\n", - " return client.chat.completions.create(model=model, messages=messages).choices[0].message" - ] + "1cfe99f3f2af48e1800ad53f911de327": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "OGT5ZjrK7HVs" - }, - "outputs": [], - "source": [ - "def generate_article(item, value, all_facts, n_retries=5):\n", - " prompt = \\\n", - "f\"\"\"\n", - "\n", - "You are in charge of writing knowledge-base articles to document important company and workplace information.\n", - "Your articles will be used as reference by the employees of the company.\n", - "\n", - "\n", - "Here is a list of facts in the knowledge base:\n", - "\n", - "{all_facts}\n", - "\n", - "\n", - "You need to write an article about this specific fact:\n", - "\n", - "The {item} is {value}.\n", - "\n", - "\n", - "\n", - "* Generate a knowledge-base article that contains the as-is. Do not modify or add to its text.\n", - "* Hide the fact inside a bunch of other related information, but do not source the related information from . Make stuff up.\n", - "* Make sure that nothing you write contradicts \n", - "* You must use HTML format. Generate only the tag.\n", - "* Don't include information about the knowledge base itself or headers like \\\"welcome to our knowledge base\\\".\n", - "\n", - "\n", - "\"\"\"\n", - " messages = [{\"role\": \"system\", \"content\": \"You are a ServiceNow system administrator in charge of writing knowledge base articles to help employees in your organization.\"},\n", - " {\"role\": \"user\", \"content\": prompt}]\n", - "\n", - " print(\"... attempting to produce article\")\n", - " for retry in range(n_retries):\n", - " try:\n", - " print(f\"... try {retry}\")\n", - " messages.append({\"role\": \"assistant\", \"content\": chat(messages).content})\n", - " article = messages[-1][\"content\"].replace(\"\", \"\").replace(\"\", \"\").replace(\"```HTML\", \"\").replace(\"```\", \"\").replace(\"\", \"\").replace(\"\", \"\").strip()\n", - "\n", - " # Validate that the fact is included without modification\n", - " assert f\"the {item} is {value}\".lower() in article.lower(), f\"Error: Could not find the string \\\"The {item} is {value}\\\" in this article.\"\n", - " print(\"... valid article found.\")\n", - " break\n", - " except AssertionError as e:\n", - " messages.append({\"role\": \"user\", \"content\": str(e) + \"\\nDon't apologize and try again.\"})\n", - "\n", - " if messages[-1][\"role\"] == \"user\":\n", - " raise RuntimeError(f\"Failed to produce a valid article after {n_retries} retries.\")\n", - "\n", - " return article\n" - ] + "483996fae22d4d2e89f83c05c7db6e76": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "BuAPHUc18EDy", - "outputId": "f58c7b3b-cdef-4c04-a946-be6f817cca80" - }, - "outputs": [], - "source": [ - "facts = [\n", - " {\"item\": \"password to conference room A-561\", \"value\": \"roo918k\"},\n", - " {\"item\": \"address of office #456\", \"value\": \"42, Pizza street, New York, USA\"},\n", - " {\"item\": \"number of employees in department X\", \"value\": \"75\"},\n", - " {\"item\": \"CEO's name\", \"value\": \"Alex Johnson\"},\n", - " {\"item\": \"year of company establishment\", \"value\": \"1998\"},\n", - " {\"item\": \"Wi-Fi network name in office #456\", \"value\": \"Office456_WiFi\"},\n", - " {\"item\": \"Wi-Fi password for office #456\", \"value\": \"456SecureNet!\"},\n", - " {\"item\": \"average annual revenue\", \"value\": \"$50 million\"},\n", - " {\"item\": \"number of branches worldwide\", \"value\": \"18\"},\n", - " {\"item\": \"name of the head of HR department\", \"value\": \"Samantha Green\"},\n", - " {\"item\": \"brand of coffee machine in kitchen #3\", \"value\": \"Delonghi Magnifica\"},\n", - " {\"item\": \"company's stock ticker symbol\", \"value\": \"COMPX\"},\n", - " {\"item\": \"company's main product\", \"value\": \"Advanced Analytics Software\"},\n", - " {\"item\": \"color of the carpet in conference room A-561\", \"value\": \"Navy Blue\"},\n", - " {\"item\": \"annual budget for marketing department\", \"value\": \"$2 million\"},\n", - " {\"item\": \"capacity of conference room B-762\", \"value\": \"30 people\"},\n", - " {\"item\": \"number of floors in the main office building\", \"value\": \"10\"},\n", - " {\"item\": \"type of plants in the lobby of office #456\", \"value\": \"Ficus Lyrata\"},\n", - " {\"item\": \"catering service provider for office events\", \"value\": \"Gourmet Caterers\"},\n", - " {\"item\": \"software used for payroll management\", \"value\": \"QuickBooks Payroll\"},\n", - " {\"item\": \"number of parking spaces in office #456 parking lot\", \"value\": \"150\"},\n", - " {\"item\": \"company's first product\", \"value\": \"Data Analysis Toolkit\"},\n", - " {\"item\": \"brand of computers used in IT department\", \"value\": \"Lenovo ThinkPad\"},\n", - " {\"item\": \"annual CSR budget\", \"value\": \"$500,000\"},\n", - " {\"item\": \"make of company's official car\", \"value\": \"Tesla Model X\"},\n", - " {\"item\": \"name of the company's auditor\", \"value\": \"PriceWaterhouseCoopers\"},\n", - " {\"item\": \"number of customer service representatives\", \"value\": \"40\"},\n", - " {\"item\": \"type of air conditioning system in office #456\", \"value\": \"Centralized HVAC\"},\n", - " {\"item\": \"name of the cafeteria manager in office #456\", \"value\": \"Miguel Torres\"},\n", - " {\"item\": \"duration of lunch break in office #456\", \"value\": \"1 hour\"},\n", - " {\"item\": \"official language for company communication\", \"value\": \"English\"},\n", - " {\"item\": \"number of conference rooms in office #456\", \"value\": \"8\"},\n", - " {\"item\": \"average time for IT support response\", \"value\": \"15 minutes\"},\n", - " {\"item\": \"brand of printers in office #456\", \"value\": \"HP LaserJet Pro\"},\n", - " {\"item\": \"capacity of the largest meeting room\", \"value\": \"50 people\"},\n", - " {\"item\": \"number of patents held by the company\", \"value\": \"35\"},\n", - " {\"item\": \"company's main competitor\", \"value\": \"TechRivals Inc.\"},\n", - " {\"item\": \"company's slogan\", \"value\": \"Innovating the Future\"},\n", - " {\"item\": \"number of countries the company operates in\", \"value\": \"12\"},\n", - " {\"item\": \"type of security system at office #456\", \"value\": \"Biometric Access Control\"},\n", - " {\"item\": \"name of the employee of the month\", \"value\": \"Elena Rodriguez\"},\n", - " {\"item\": \"brand of the photocopier in office #456\", \"value\": \"Canon ImageRunner\"},\n", - " {\"item\": \"type of coffee provided in the kitchen\", \"value\": \"Arabica Beans\"},\n", - " {\"item\": \"annual IT budget\", \"value\": \"$1.5 million\"},\n", - " {\"item\": \"name of the legal counsel firm\", \"value\": \"Baker & McKenzie\"},\n", - " {\"item\": \"average employee satisfaction score\", \"value\": \"8.5/10\"},\n", - " {\"item\": \"number of departments in the company\", \"value\": \"12\"},\n", - " {\"item\": \"company's largest client\", \"value\": \"GlobalTech Industries\"},\n", - " {\"item\": \"type of gym equipment in fitness center\", \"value\": \"Life Fitness machines\"},\n", - " {\"item\": \"average number of yearly hires\", \"value\": \"100\"},\n", - " {\"item\": \"brand of air purifiers used in the office\", \"value\": \"Dyson Pure Cool\"},\n", - " {\"item\": \"name of the employee health insurance provider\", \"value\": \"BlueCross BlueShield\"},\n", - " {\"item\": \"number of projects completed last year\", \"value\": \"22\"},\n", - " {\"item\": \"company's main export market\", \"value\": \"European Union\"},\n", - " {\"item\": \"name of the office cleaning service provider\", \"value\": \"CleanSweep Inc.\"},\n", - " {\"item\": \"type of video conferencing software used\", \"value\": \"Zoom\"},\n", - " {\"item\": \"color of the company logo\", \"value\": \"Royal Blue and Silver\"},\n", - " {\"item\": \"average client retention rate\", \"value\": \"85%\"},\n", - " {\"item\": \"name of the company's largest shareholder\", \"value\": \"David Thompson\"},\n", - " {\"item\": \"total square footage of office #456\", \"value\": \"25,000 square feet\"},\n", - " {\"item\": \"type of backup power system in office #456\", \"value\": \"Diesel Generators\"},\n", - " {\"item\": \"average yearly expenditure on office supplies\", \"value\": \"$80,000\"},\n", - " {\"item\": \"name of the cafeteria food supplier\", \"value\": \"FreshFoods Ltd.\"},\n", - " {\"item\": \"type of fire safety system in office #456\", \"value\": \"Automatic Sprinkler System\"},\n", - " {\"item\": \"company's primary industry\", \"value\": \"Technology\"},\n", - " {\"item\": \"number of annual company-wide meetings\", \"value\": \"4\"},\n", - " {\"item\": \"brand of smartphones provided to employees\", \"value\": \"Samsung Galaxy\"},\n", - " {\"item\": \"name of the software used for project management\", \"value\": \"Trello\"},\n", - " {\"item\": \"average number of business trips per employee annually\", \"value\": \"3\"},\n", - " {\"item\": \"number of IT support staff\", \"value\": \"15\"},\n", - " {\"item\": \"company's primary social media platform for marketing\", \"value\": \"LinkedIn\"},\n", - " {\"item\": \"number of active contracts\", \"value\": \"40\"},\n", - " {\"item\": \"annual expenditure on R&D\", \"value\": \"$3 million\"},\n", - " {\"item\": \"type of chairs used in conference room A-561\", \"value\": \"Ergonomic Office Chairs\"},\n", - " {\"item\": \"name of the most used software by the design team\", \"value\": \"Adobe Creative Suite\"},\n", - " {\"item\": \"average delivery time for company products\", \"value\": \"5 business days\"},\n", - " {\"item\": \"type of lighting in office #456\", \"value\": \"LED Lights\"},\n", - " {\"item\": \"average monthly electricity bill for office #456\", \"value\": \"$10,000\"},\n", - " {\"item\": \"company's largest expense category\", \"value\": \"Employee Salaries\"},\n", - " {\"item\": \"brand of the refrigerator in the kitchen\", \"value\": \"LG InstaView\"},\n", - " {\"item\": \"type of health and safety training provided\", \"value\": \"First Aid and Fire Safety\"},\n", - " {\"item\": \"name of the corporate social responsibility program\", \"value\": \"TechForGood\"},\n", - " {\"item\": \"annual water consumption in office #456\", \"value\": \"50,000 gallons\"},\n", - " {\"item\": \"company's policy on remote work\", \"value\": \"Hybrid Model\"},\n", - " {\"item\": \"number of customer complaints last year\", \"value\": \"120\"},\n", - " {\"item\": \"name of the most popular product\", \"value\": \"Smart Analytics Pro\"},\n", - " {\"item\": \"type of snacks available in the kitchen\", \"value\": \"Healthy and Organic Snacks\"},\n", - " {\"item\": \"average duration of employee tenure\", \"value\": \"5 years\"},\n", - " {\"item\": \"brand of the security cameras in office #456\", \"value\": \"Axis Communications\"},\n", - " {\"item\": \"type of heating system in office #456\", \"value\": \"Forced Air Heating\"},\n", - " {\"item\": \"name of the internet service provider for office #456\", \"value\": \"Comcast Xfinity\"},\n", - " {\"item\": \"company's policy on environmental sustainability\", \"value\": \"Zero-Waste Initiatives\"},\n", - " {\"item\": \"name of the annual team-building retreat location\", \"value\": \"Lakeview Resort\"},\n", - " {\"item\": \"type of vending machines in office #456\", \"value\": \"Cashless Payment Vending\"},\n", - " {\"item\": \"number of team leaders in the company\", \"value\": \"35\"},\n", - " {\"item\": \"annual spending on employee training programs\", \"value\": \"$250,000\"},\n", - " {\"item\": \"CEO's direct phone line\", \"value\": \"+1-555-1010-2020\"},\n", - " {\"item\": \"Wi-Fi network name in the lobby\", \"value\": \"CorporateGuest123\"},\n", - " {\"item\": \"last fire drill date at main building\", \"value\": \"2023-06-15\"},\n", - " {\"item\": \"number of employees in marketing department\", \"value\": \"35\"},\n", - "]\n", - "\n", - "print(\"Number of facts:\", len(facts))\n", - "\n", - "for i, f in enumerate(facts):\n", - " f[\"article\"] = generate_article(f[\"item\"], f[\"value\"], all_facts=facts)\n", - " print(\"... Article\", i + 1, f[\"item\"], f[\"value\"])\n", - " print(f[\"article\"])\n", - " print(\"\\n\" * 2)\n" - ] + "4c8d2b15f0d84fe3b7adc5feeae9b6db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_571e7187a94d4860915f70f85654573b", + "placeholder": "​", + "style": "IPY_MODEL_16997d3f7a414ec79bcca88298fac743", + "value": " 100/100 [15:04<00:00, 4.18s/it]" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 262 - }, - "id": "tDTelvGXWp-P", - "outputId": "ddb6f999-f127-4e25-bb9f-56037a4fb7ed" - }, - "outputs": [], - "source": [ - "for f in facts:\n", - " f[\"article\"] = f[\"article\"].replace(\"```HTML\", \"\").replace(\"```\", \"\").replace(\"\", \"\").replace(\"\", \"\").strip()\n", - "\n", - "json.dump(facts, open(\"knowledge_base.json\", \"w\"))" - ] + "511c96890e1a4989bae8390ca627ac37": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "ctj7CPtAon3w" - }, - "source": [ - "# Generate questions and answers" - ] + "571e7187a94d4860915f70f85654573b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QrPIz4CZ3ier", - "outputId": "ceb47583-81ea-4f21-9fa5-dff4045d16c9" - }, - "outputs": [], - "source": [ - "import concurrent.futures\n", - "import re\n", - "\n", - "from collections import Counter\n", - "from tqdm.notebook import tqdm\n", - "\n", - "\n", - "# Reload the generated KB\n", - "kb = json.load(open(\"knowledge_base.json\", \"r\"))\n", - "print(\"Loaded\", len(kb), \"articles\")\n", - "\n", - "\n", - "# Clear all questions\n", - "for kb_i in kb:\n", - " if \"questions\" in kb_i:\n", - " kb_i[\"questions\"] = []\n", - " kb_i[\"alternative_answers\"] = []" - ] + "5f147ed1bb86428397e25e5f4bbf14f4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ca30f6701b8c44409a5c7356c264e3e6", + "IPY_MODEL_d304373e7bef447f919b083a64547a58", + "IPY_MODEL_4c8d2b15f0d84fe3b7adc5feeae9b6db" + ], + "layout": "IPY_MODEL_f82189b7bd4944c0999ac01600efa4ae" + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ee_KMC2o3gDD" - }, - "source": [ - "## Generate several variants of the question" - ] + "6bbe2aa489d6427daca0d0ac04a51f6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000, - "referenced_widgets": [ - "e77eceb6a8624f66ac12cc3df788b37e", - "ddb341b6b65a4ab19da7c1811a706be7", - "b2de9413f61b4be7947f9bc0d3f12f41", - "dd4959d2ace04d5eb2a8c3274a069647", - "be1c8e816d0743eaa5c49d26e0050e94", - "6bbe2aa489d6427daca0d0ac04a51f6e", - "1cfe99f3f2af48e1800ad53f911de327", - "d7c95fe313c546a6af713d3cb0e004d6", - "a1221b65b0644cd794dc477d0492979f", - "511c96890e1a4989bae8390ca627ac37", - "0e044725d805479b9038d0552f277a7e" - ] - }, - "id": "NLERyQk-Xge5", - "outputId": "0fb2d008-ac09-4cdf-f63d-cc623df09b63" - }, - "outputs": [], - "source": [ - "\n", - "def get_most_frequent_items(my_list):\n", - " # Count the frequency of each item in the list\n", - " frequency = Counter(my_list)\n", - "\n", - " # Find the maximum frequency\n", - " max_frequency = max(frequency.values())\n", - "\n", - " # Get all items with the maximum frequency\n", - " most_frequent = [item for item, count in frequency.items() if count == max_frequency]\n", - "\n", - " return most_frequent\n", - "\n", - "\n", - "def generate_question(article, item, value, n_questions=10, n_retries=5):\n", - " prompt = f\"\"\"\n", - " Here is an article taken from a company knowledge base.\n", - "\n", - "
\n", - " {article}\n", - "
\n", - "\n", - " This article contains the following fact:\n", - " \n", - " The {item} is {value}.\n", - " \n", - "\n", - " Here is a question about this fact to which the answer is \\\"{value}\\\":\n", - " \n", - " What is the {item}?\n", - " \n", - "\n", - " Produce {n_questions} rephrasings of this question.\n", - "\n", - " \n", - " * Make sure that your questions are precise and unambiguous.\n", - " * It must be clear that the questions are asking about \\\"{item}\\\" and\n", - " their answer must still be exactly \\\"{value}\\\".\n", - " * Make sure that you provide clear and specific instructions on the expected\n", - " format for the answer (e.g., Day, Month, Year).\n", - " * You cannot, in any circumstances, reveal information from the answer in the question or instructions.\n", - " * Make sure they are questions that end with a question mark.\n", - " * Make sure that your questions do not mention the article (e.g., \\\"in the article\\\").\n", - " * Answer with one per line and do not number them.\n", - " \n", - "\n", - " \n", - " Suppose the question is \\\"On which day was the company founded?\\\" and the answer is \\\"January 26, 2024\\\",\n", - " then a good rephrasing would be \\\"When was the company founded? Answer with Month Day, Year\\\".\n", - " \n", - "\n", - " \n", - " Suppose the question is \\\"What kind of fridge do we have in the cafeteria?\\\" and the answer is \\\"LG X456\\\",\n", - " then a good rephrasing would be \\\"Which type of fridge is in the cafeteria? Answer with Brand followed by Model name\\\".\n", - " \n", - "\n", - " \n", - " Suppose the question is \\\"Where is the company headquarter located?\\\" and the answer is \\\"123, Banana Street, Montreal, Canada\\\",\n", - " then a good rephrasing would be \\\"What's the address of the company headquarter? Answer with Number, Street, City, Country\\\".\n", - " \n", - "\n", - " \n", - " Suppose the question is \\\"What were the total sales of the company in 2023?\\\" and the answer is \\\"150B$\\\",\n", - " then a good rephrasing would be \\\"What do the company sales for 2023 sum up to? Answer with NumberB$, where B is billions\\\".\n", - " \n", - " \"\"\"\n", - " good_questions = set([])\n", - " messages = [{\"role\": \"user\", \"content\": prompt}]\n", - " while len(good_questions) < n_questions:\n", - " try:\n", - " # Generate questions\n", - " questions = chat(messages).content\n", - "\n", - " # Parse output\n", - " # ... check the article is not mentioned\n", - " assert \"article\" not in questions.lower(), \"You are not allowed to mention the article in your rephrasing of the questions.\"\n", - " # ... check the number of actual questions included\n", - " assert questions.count(\"?\") == n_questions, f\"I couldn't find {n_questions} question marks in the output. Make sure all questions end with ?\"\n", - " # ... heuristic to detect numbered questions\n", - " assert not all(f\"{x}.\" in questions for x in range(4)), \"The questions appear to be numbered, but they should not.\"\n", - " questions = [q.strip() for q in questions.split(\"\\n\") if q != \"\"]\n", - " # ... check one question per line\n", - " assert len(questions) == n_questions, f\"Your answer is not {n_questions} lines long. Make sure it contains {n_questions} questions and one per line.\"\n", - " # ... check each question has instructions\n", - " assert all(len(q.split(\"?\")) > 1 and len(\"\".join(q.split(\"?\")[1:]).strip()) > 5 for q in questions), f\"Make sure you provide clear formatting instructions for each question (e.g., \\\"Question? Answer with\\\").\"\n", - " # ... check that answer is not mentioned\n", - " for q in questions:\n", - " error = \"Do not include the answer in the questions/instructions!\"\n", - " # ... exact value is not in the question\n", - " assert value.lower() not in q.lower(), error\n", - "\n", - " # Validate questions\n", - " bad_questions = []\n", - " bad_answers = []\n", - " for q in questions:\n", - " print(\"... testing:\", q, end=\" \")\n", - " success, answers = is_question_answerable(article, q, value)\n", - " if not success:\n", - " bad_questions.append(q)\n", - " bad_answers.append(answers)\n", - " print(\"FAIL\")\n", - " print(\"... preemptively stopping to give feedback\")\n", - " break\n", - " else:\n", - " good_questions.add(q)\n", - " print(\"PASS\")\n", - "\n", - " # Give feedback and retry\n", - " if len(bad_questions) > 0:\n", - " feedback = \"\"\n", - " for q, a in zip(bad_questions, bad_answers):\n", - " feedback += \"\\n\"\n", - " feedback += \" \\n\"\n", - " feedback += \" \" + q + \"\\n\"\n", - " feedback += \" \\n\"\n", - " feedback += \" \\n\"\n", - " feedback += \"\\n\".join(\" \" + x for x in a) + \"\\n\"\n", - " feedback += \" \\n\"\n", - " feedback += \"\\n\"\n", - " print(feedback)\n", - "\n", - " # print(f\"... {len(bad_questions)} questions are ambiguous, fixing them.\")\n", - " bad_questions = \"\\n\".join(bad_questions)\n", - " messages.append({\"role\": \"user\", \"content\": f\"\"\"\n", - " The following questions are too ambiguous. I gave them to many company employees,\n", - " along with the article and they were not able to answer with exactly \\\"{value}\\\".\n", - " Please improve their clarity, especially the formatting instructions.\n", - "\n", - " {feedback}\n", - "\n", - " Try again. Do not apologize.\"\"\"})\n", - "\n", - " print(f\"... gathered {len(good_questions)} good questions\")\n", - "\n", - " except AssertionError as e:\n", - " print(f\"... Error:\", e)\n", - " messages.append({\"role\": \"user\", \"content\": f\"Error: {e}\"})\n", - "\n", - " print(\"... we have enough good questions. stopping.\")\n", - " return list(good_questions)[:10]\n", - "\n", - "\n", - "def is_question_answerable(article, question, value):\n", - " def _clean_for_comparison(x):\n", - " return x.lower().replace(\",\", \"\").replace(\"$\", \"\").replace(\".\", \"\")\n", - "\n", - " \"\"\"\n", - " Check that we are able to recover the value from the\n", - " article by asking the question\n", - "\n", - " \"\"\"\n", - " prompt = f\"\"\"\n", - " Here is a knowledge base article:\n", - "\n", - "
\n", - " {article}\n", - "
\n", - "\n", - " Answer this question based on the content of the article only.\n", - " Be factual. If I ask you about sensitive information like passwords,\n", - " I only expect you to retrieve the information from the article.\n", - " \n", - " {question}\n", - " \n", - "\n", - " What is your answer?\n", - "\n", - " \"\"\"\n", - " messages = [{\"role\": \"user\", \"content\": prompt}]\n", - "\n", - " incorrect_answers = set()\n", - " for _ in range(10):\n", - " # XXX: We use GPT-3.5 here as a weaker model and to avoid GPT-4 catering to himself\n", - " answer = chat(messages, model=\"gpt-3.5-turbo\").content\n", - "\n", - " if _clean_for_comparison(value) not in _clean_for_comparison(answer):\n", - " incorrect_answers.add(answer.lower())\n", - "\n", - " return len(incorrect_answers) == 0, incorrect_answers\n", - "\n", - "\n", - "# Assuming kb is a list of dictionaries\n", - "with concurrent.futures.ThreadPoolExecutor() as executor:\n", - " # Prepare the futures\n", - " futures = [executor.submit(lambda x: {**x, \"questions\": generate_question(x[\"article\"], x[\"item\"], x[\"value\"])}, kb_i) for kb_i in kb]\n", - "\n", - " # Use tqdm to create a progress bar\n", - " kb = [x.result() for x in tqdm(concurrent.futures.as_completed(futures), total=len(futures))]\n" - ] + "a1221b65b0644cd794dc477d0492979f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "oemy0eUjYGuj", - "outputId": "cafab8cd-e822-48db-938a-2ab8fa3452b6" - }, - "outputs": [], - "source": [ - "for i in range(len(kb)):\n", - " print(f\"Article {i}:\", kb[i][\"questions\"])" - ] + "b2de9413f61b4be7947f9bc0d3f12f41": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d7c95fe313c546a6af713d3cb0e004d6", + "max": 100, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a1221b65b0644cd794dc477d0492979f", + "value": 100 + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "Re4HGFh6ow7-" - }, - "source": [ - "## Generate a few alternative answers" - ] + "be1c8e816d0743eaa5c49d26e0050e94": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 66, - "referenced_widgets": [ - "5f147ed1bb86428397e25e5f4bbf14f4", - "ca30f6701b8c44409a5c7356c264e3e6", - "d304373e7bef447f919b083a64547a58", - "4c8d2b15f0d84fe3b7adc5feeae9b6db", - "f82189b7bd4944c0999ac01600efa4ae", - "e69716214e49423898fdc173108cadbe", - "483996fae22d4d2e89f83c05c7db6e76", - "fdd79cc467e44d849d31b43b416bcb20", - "eba164765ad544c38a7679d494343551", - "571e7187a94d4860915f70f85654573b", - "16997d3f7a414ec79bcca88298fac743" - ] - }, - "id": "Dxjc0HQOo25w", - "outputId": "fdbd7723-1dda-4eda-f0ee-281fc051d04c" - }, - "outputs": [], - "source": [ - "def alternative_answers(article, item, value, questions, n_answers=10):\n", - " questions = \"\\n\".join(questions)\n", - "\n", - " prompt = f\"\"\"\n", - " Here is a set of questions:\n", - " \n", - " {questions}\n", - " \n", - "\n", - " The exact answer to all of these questions is:\n", - " \n", - " {value}\n", - " \n", - "\n", - " Give me {n_answers} other ways to spell out the answer.\n", - " Don't add context words around it or anything, just reformulate it.\n", - "\n", - " \n", - " Initial value: 5.5/10\n", - " Reformulated: 5.5 out of 10\n", - " Reformulated: 55%\n", - " Reformulated: fifty-five percent\n", - " \n", - "\n", - " \n", - " Initial value: Tesla Model X\n", - " Reformulated: Model X by Tesla\n", - " Reformulated: Tesla's Model X\n", - " \n", - "\n", - " \n", - " Initial value: 150$\n", - " Reformulated: one hundred fifty dollars\n", - " Reformulated: 150.00$\n", - " Reformulated: $150\n", - " \n", - "\n", - " Answer with one per line. Don't number them.\n", - "\n", - " \"\"\"\n", - " messages = [{\"role\": \"user\", \"content\": prompt}]\n", - "\n", - " while True:\n", - " try:\n", - " answers = chat(messages).content\n", - "\n", - " # Validation\n", - " # ... heuristic to detect numbered questions\n", - " assert not all(f\"{x}.\" in answers for x in range(4)), \"The answers appear to be numbered, but they should not.\"\n", - " answers = [a.strip() for a in answers.split(\"\\n\") if a != \"\"]\n", - " # ... check one question per line\n", - " assert len(answers) == n_answers, f\"Your response is not {answers} lines long. Make sure it contains {answers} answers and one per line.\"\n", - " assert len(set(answers)) == n_answers, f\"You provided duplicate values. There were only {len(set(answers))} unique values.\"\n", - " break\n", - " except AssertionError as e:\n", - " print(f\"... Error:\", e)\n", - " messages.append({\"role\": \"user\", \"content\": f\"Error: {e}\"})\n", - "\n", - " return answers\n", - "\n", - "\n", - "for kb_i in tqdm(kb, total=len(kb)):\n", - " kb_i[\"alternative_answers\"] = alternative_answers(kb_i[\"article\"], kb_i[\"item\"], kb_i[\"value\"], kb_i[\"questions\"])" - ] + "ca30f6701b8c44409a5c7356c264e3e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e69716214e49423898fdc173108cadbe", + "placeholder": "​", + "style": "IPY_MODEL_483996fae22d4d2e89f83c05c7db6e76", + "value": "100%" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "CFi8jwUhFUDP", - "outputId": "08a3c5c4-5da9-4fc4-a443-0e2ac1891f7e" - }, - "outputs": [], - "source": [ - "for kb_i in kb:\n", - " print(kb_i[\"alternative_answers\"])" - ] + "d304373e7bef447f919b083a64547a58": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdd79cc467e44d849d31b43b416bcb20", + "max": 100, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_eba164765ad544c38a7679d494343551", + "value": 100 + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "i-CbZL-do1fX" - }, - "source": [ - "## Save it" - ] + "d7c95fe313c546a6af713d3cb0e004d6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Uww5Ut2rfR4f" - }, - "outputs": [], - "source": [ - "json.dump(kb, open(\"knowledge_base.json\", \"w\"))" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] + "dd4959d2ace04d5eb2a8c3274a069647": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_511c96890e1a4989bae8390ca627ac37", + "placeholder": "​", + "style": "IPY_MODEL_0e044725d805479b9038d0552f277a7e", + "value": " 100/100 [23:30<00:00, 59.58s/it]" + } + }, + "ddb341b6b65a4ab19da7c1811a706be7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6bbe2aa489d6427daca0d0ac04a51f6e", + "placeholder": "​", + "style": "IPY_MODEL_1cfe99f3f2af48e1800ad53f911de327", + "value": "100%" + } + }, + "e69716214e49423898fdc173108cadbe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" + "e77eceb6a8624f66ac12cc3df788b37e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ddb341b6b65a4ab19da7c1811a706be7", + "IPY_MODEL_b2de9413f61b4be7947f9bc0d3f12f41", + "IPY_MODEL_dd4959d2ace04d5eb2a8c3274a069647" + ], + "layout": "IPY_MODEL_be1c8e816d0743eaa5c49d26e0050e94" + } }, - "language_info": { - "name": "python" + "eba164765ad544c38a7679d494343551": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "0e044725d805479b9038d0552f277a7e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "16997d3f7a414ec79bcca88298fac743": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "1cfe99f3f2af48e1800ad53f911de327": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "483996fae22d4d2e89f83c05c7db6e76": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "4c8d2b15f0d84fe3b7adc5feeae9b6db": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_571e7187a94d4860915f70f85654573b", - "placeholder": "​", - "style": "IPY_MODEL_16997d3f7a414ec79bcca88298fac743", - "value": " 100/100 [15:04<00:00, 4.18s/it]" - } - }, - "511c96890e1a4989bae8390ca627ac37": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "571e7187a94d4860915f70f85654573b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5f147ed1bb86428397e25e5f4bbf14f4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_ca30f6701b8c44409a5c7356c264e3e6", - "IPY_MODEL_d304373e7bef447f919b083a64547a58", - "IPY_MODEL_4c8d2b15f0d84fe3b7adc5feeae9b6db" - ], - "layout": "IPY_MODEL_f82189b7bd4944c0999ac01600efa4ae" - } - }, - "6bbe2aa489d6427daca0d0ac04a51f6e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a1221b65b0644cd794dc477d0492979f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "b2de9413f61b4be7947f9bc0d3f12f41": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d7c95fe313c546a6af713d3cb0e004d6", - "max": 100, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a1221b65b0644cd794dc477d0492979f", - "value": 100 - } - }, - "be1c8e816d0743eaa5c49d26e0050e94": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ca30f6701b8c44409a5c7356c264e3e6": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e69716214e49423898fdc173108cadbe", - "placeholder": "​", - "style": "IPY_MODEL_483996fae22d4d2e89f83c05c7db6e76", - "value": "100%" - } - }, - "d304373e7bef447f919b083a64547a58": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fdd79cc467e44d849d31b43b416bcb20", - "max": 100, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_eba164765ad544c38a7679d494343551", - "value": 100 - } - }, - "d7c95fe313c546a6af713d3cb0e004d6": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "dd4959d2ace04d5eb2a8c3274a069647": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_511c96890e1a4989bae8390ca627ac37", - "placeholder": "​", - "style": "IPY_MODEL_0e044725d805479b9038d0552f277a7e", - "value": " 100/100 [23:30<00:00, 59.58s/it]" - } - }, - "ddb341b6b65a4ab19da7c1811a706be7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6bbe2aa489d6427daca0d0ac04a51f6e", - "placeholder": "​", - "style": "IPY_MODEL_1cfe99f3f2af48e1800ad53f911de327", - "value": "100%" - } - }, - "e69716214e49423898fdc173108cadbe": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e77eceb6a8624f66ac12cc3df788b37e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_ddb341b6b65a4ab19da7c1811a706be7", - "IPY_MODEL_b2de9413f61b4be7947f9bc0d3f12f41", - "IPY_MODEL_dd4959d2ace04d5eb2a8c3274a069647" - ], - "layout": "IPY_MODEL_be1c8e816d0743eaa5c49d26e0050e94" - } - }, - "eba164765ad544c38a7679d494343551": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "f82189b7bd4944c0999ac01600efa4ae": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fdd79cc467e44d849d31b43b416bcb20": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - } - } + "f82189b7bd4944c0999ac01600efa4ae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fdd79cc467e44d849d31b43b416bcb20": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } } - }, - "nbformat": 4, - "nbformat_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/scripts/make_human_eval_curriculum.py b/scripts/make_human_eval_curriculum.py new file mode 100644 index 0000000..ae4a7eb --- /dev/null +++ b/scripts/make_human_eval_curriculum.py @@ -0,0 +1,44 @@ +import random + +from browsergym.workarena import get_all_tasks_humans + + +tasks_l2 = get_all_tasks_humans(filter="l2", meta_seed=42) +tasks_l3 = get_all_tasks_humans(filter="l3", meta_seed=42) + +tasks = tasks_l2 + tasks_l3 +random.shuffle(tasks) + +annotators = [ + "darwiche", + "parikh", + "marchand", + "paquet", + "nayal", + "huang", + "subbaraj", + "williams", + "li", + "marcotte", + "rancourt", + "prince_tremblay", + "ashok", + "bajaj", +] +random.shuffle(annotators) + +print("Number of tasks: ", len(tasks)) +print("Number of annotators: ", len(annotators)) + +tasks_by_annotator = {} +n_base_assignment = len(tasks) // len(annotators) +n_extra_assignment = len(tasks) % len(annotators) +for i, annotator in enumerate(annotators): + n_assignments = n_base_assignment + (1 if i < n_extra_assignment else 0) + tasks_by_annotator[annotator] = tasks[:n_assignments] + tasks = tasks[n_assignments:] + print(f"{annotator}: {n_assignments}") + + with open(f"{annotator}.tasks", "w") as f: + for task in tasks_by_annotator[annotator]: + f.write(f"{task[0].__name__},{task[1]}\n") diff --git a/scripts/wa_action_traces.py b/scripts/wa_action_traces.py new file mode 100644 index 0000000..8ff5b84 --- /dev/null +++ b/scripts/wa_action_traces.py @@ -0,0 +1,131 @@ +""" +A demonstration of how action traces (with observations) can be extracted +for WorkArena tasks without modifying the task code. + +Author: Alexandre Drouin (alexandre.drouin@servicenow.com) + +Notes: +- This approach relies on monkey patching the playwright actions to log the actions and observations. + It has not been tested for parallel execution. It might work with multiprocessing, but it will for + sure not work with multithreading. + +""" + +import importlib +import logging +import os +import pickle +import playwright.sync_api as playwright_sync + +from browsergym.core.env import BrowserEnv +from browsergym.workarena import ALL_WORKARENA_TASKS +from collections import defaultdict +from tenacity import retry, stop_after_attempt, wait_fixed +from time import time + + +N_PER_TASK = 10 + + +def monkey_patch_playwright(observation_callback, trace_storage): + """ + A function that overrides the default playwright actions to log the actions and observations. + + Parameters: + ------------ + observation_callback: callable + A function that returns the observation of the environment. + trace_storage: list + A list to store the trace of the actions and observations. + These will be appended in-place. + + """ + + def wrapper(func, interface): + def wrapped(*args, **kwargs): + # Get the observation + obs = observation_callback() + + # Get the BID of the element on which we are acting. + if interface.__name__ == "Locator": + # Get the locator + locator = args[0] + # Get the BID + bid = locator.element_handle().evaluate('(el) => el.getAttribute("bid")') + elif interface.__name__ == "Keyboard": + # Get the BID of the element + bid = "keyboard" + else: + # Get the BID of the element + bid = args[0].evaluate('(el) => el.getAttribute("bid")') + + logging.info(f"Action: {func.__name__} BID: {bid} -- Args: {args[1:]} {kwargs}") + trace_storage.append( + { + "obs": obs, + "action": func.__name__, + "args": args[1:], + "kwargs": kwargs, + "bid": bid, + "time": time(), + } + ) + + # Resume action + return func(*args, **kwargs) + + return wrapped + + # Interfaces and actions we want to monkey patch + importlib.reload(playwright_sync) + from playwright.sync_api import Page, Frame, Locator, Keyboard, ElementHandle + + # TODO: Make sure the list of interfaces and actions is exhaustive + # It covers all that is used in WorkArena cheats as of April 11, 2024 + interfaces = [Page, Frame, Locator, Keyboard, ElementHandle] + actions = ["click", "select_option", "set_checked", "fill", "press", "type", "down", "up"] + + for interface in interfaces: + for action in actions: + if hasattr(interface, action): + setattr(interface, action, wrapper(getattr(interface, action), interface)) + print(f"Monkey patched {interface.__name__}.{action}") + + +@retry(stop=stop_after_attempt(3), wait=wait_fixed(2)) +def extract_trace(task_cls, headless=True): + """ + Extracts the trace of actions and observations for a given task. + + Parameters: + ------------ + task_cls: class + The class of the task to extract the trace from. + + """ + # Instantiate a new environment + env = BrowserEnv(task_entrypoint=task_cls, headless=headless, slow_mo=1000) + + # Setup customized tracing + trace = [] + monkey_patch_playwright(observation_callback=env._get_obs, trace_storage=trace) + + env.reset() + env.task.cheat(env.page, env.chat.messages) + env.close() + + return trace + + +if __name__ == "__main__": + os.makedirs("trace_profiling", exist_ok=True) + + task_traces = defaultdict(list) + for task in ALL_WORKARENA_TASKS: + print("Task:", task) + for i in range(N_PER_TASK): + print(f"Extracting trace {i+1}/{N_PER_TASK}") + trace = extract_trace(task, headless=True) + task_traces[task].append(trace) + + pickle.dump(task_traces, open("trace_profiling/task_traces.pkl", "wb")) diff --git a/src/browsergym/workarena/__init__.py b/src/browsergym/workarena/__init__.py index 5ced43c..1444d17 100644 --- a/src/browsergym/workarena/__init__.py +++ b/src/browsergym/workarena/__init__.py @@ -1,26 +1,137 @@ -__version__ = "0.2.1" +__version__ = "0.3.0dev" + +import inspect +import numpy as np from browsergym.core.registration import register_task +from .tasks.comp_building_block import CompositionalBuildingBlockTask from .tasks.dashboard import __TASKS__ as DASHBOARD_TASKS from .tasks.form import __TASKS__ as FORM_TASKS from .tasks.knowledge import __TASKS__ as KB_TASKS from .tasks.list import __TASKS__ as LIST_TASKS from .tasks.navigation import __TASKS__ as NAVIGATION_TASKS +from .tasks.compositional.base import CompositionalTask +from .tasks.compositional.update_task import __TASKS__ as UPDATE_TASKS +from .tasks.compositional import ( + ALL_COMPOSITIONAL_TASKS, + ALL_COMPOSITIONAL_TASKS_L2, + ALL_COMPOSITIONAL_TASKS_L3, + AGENT_CURRICULUM_L2, + AGENT_CURRICULUM_L3, + HUMAN_CURRICULUM_L2, + HUMAN_CURRICULUM_L3, +) +from .tasks.compositional.base import HumanEvalTask from .tasks.service_catalog import __TASKS__ as SERVICE_CATALOG_TASKS ALL_WORKARENA_TASKS = [ + *ALL_COMPOSITIONAL_TASKS_L2, + *ALL_COMPOSITIONAL_TASKS_L3, *DASHBOARD_TASKS, *FORM_TASKS, *KB_TASKS, *LIST_TASKS, *NAVIGATION_TASKS, *SERVICE_CATALOG_TASKS, + *UPDATE_TASKS, +] +ATOMIC_TASKS = [ + task + for task in ALL_WORKARENA_TASKS + if inspect.isclass(task) + and not issubclass(task, CompositionalTask) + and not issubclass(task, CompositionalBuildingBlockTask) ] + # register the WorkArena benchmark for task in ALL_WORKARENA_TASKS: register_task( task.get_task_id(), task, ) + + +def get_all_tasks_agents(filter="l2", meta_seed=42, n_seed_l1=10): + all_task_tuples = [] + filter = filter.split(".") + if len(filter) > 2: + raise Exception("Unsupported filter used.") + if len(filter) == 1: + level = filter[0] + if level not in ["l1", "l2", "l3"]: + raise Exception("Unsupported category of tasks.") + else: + rng = np.random.RandomState(meta_seed) + if level == "l1": + for task in ATOMIC_TASKS: + for seed in rng.randint(0, 1000, n_seed_l1): + all_task_tuples.append((task, int(seed))) + + return all_task_tuples + + if len(filter) == 2: + level, filter_category = filter[0], filter[1] + if filter_category not in list(AGENT_CURRICULUM_L2.keys()): + raise Exception("Unsupported category of tasks.") + else: + filter_category = None + + if level == "l2": + ALL_COMPOSITIONAL_TASKS_CATEGORIES = AGENT_CURRICULUM_L2 + else: + ALL_COMPOSITIONAL_TASKS_CATEGORIES = AGENT_CURRICULUM_L3 + + for category, items in ALL_COMPOSITIONAL_TASKS_CATEGORIES.items(): + if filter_category and category != filter_category: + continue + for curr_seed in rng.randint(0, 1000, items["num_seeds"]): + random_gen = np.random.RandomState(curr_seed) + for task_set, count in zip(items["buckets"], items["weights"]): + tasks = random_gen.choice(task_set, count, replace=False) + for task in tasks: + all_task_tuples.append((task, curr_seed)) + + return all_task_tuples + + +def get_all_tasks_humans(filter="l2", meta_seed=42): + OFFSET = 42 + all_task_tuples = [] + filter = filter.split(".") + if len(filter) > 2: + raise Exception("Unsupported filter used.") + if len(filter) == 1: + level = filter[0] + if level not in ["l1", "l2", "l3"]: + raise Exception("Unsupported category of tasks.") + else: + rng = np.random.RandomState(meta_seed) + if level == "l1": + return [(task, rng.randint(0, 1000)) for task in ATOMIC_TASKS] + + if len(filter) == 2: + level, filter_category = filter[0], filter[1] + if filter_category not in list(HUMAN_CURRICULUM_L2.keys()): + raise Exception("Unsupported category of tasks.") + else: + filter_category = None + + if level == "l2": + ALL_COMPOSITIONAL_TASKS_CATEGORIES = HUMAN_CURRICULUM_L2 + else: + ALL_COMPOSITIONAL_TASKS_CATEGORIES = HUMAN_CURRICULUM_L3 + + for category, items in ALL_COMPOSITIONAL_TASKS_CATEGORIES.items(): + if filter_category and category != filter_category: + continue + # We will come back to this after the submission + for curr_seed in rng.randint(0, 1000, items["num_seeds"]): + random_gen = np.random.RandomState(curr_seed) + for task_set, count in zip(items["buckets"], items["weights"]): + tasks = random_gen.choice(task_set, count, replace=False) + for task in tasks: + all_task_tuples.append((task, curr_seed)) + + return all_task_tuples diff --git a/src/browsergym/workarena/api/category.py b/src/browsergym/workarena/api/category.py new file mode 100644 index 0000000..b9ab1b7 --- /dev/null +++ b/src/browsergym/workarena/api/category.py @@ -0,0 +1,74 @@ +import warnings +from faker import Faker + +fake = Faker() + +from .utils import table_api_call + +from ..instance import SNowInstance + + +def create_category( + instance: SNowInstance, + list_name: str, + category_name: str = None, +) -> list[str]: + """ + NOTE: This function creates a new category in the given list. Because categories are in a drop-down list, adding more + categories will make the list longer and this will affect the difficulty of the task. Use only if you are certain you know + what you are doing. + + Create a category for a given list + + Parameters: + ----------- + instance: SNowInstance + The instance to create the category in + list_name: str + The name of the list to create the category for (e.g. problem, incident, etc.) + category_name: str + The name of the category to create, defaults to a random category name + + Returns: + -------- + category_name, sys_id + + """ + warnings.warn( + "This function creates a new category in the given list. Because categories are in a drop-down list, adding more " + "categories will make the list longer and this will affect the difficulty of the task. Use only if you are certain you know " + "what you are doing.", + UserWarning, + ) + + if category_name is None: + category_name = fake.word() + "-" + fake.word() + + # Create category + category_data = { + "name": list_name, + "element": "category", + "value": category_name, + } + result = table_api_call( + instance=instance, + table="sys_choice", + json=category_data, + method="POST", + wait_for_record=True, + )["result"] + + sys_id = result["sys_id"] + + return category_name, sys_id + + +def get_categories(instance, list_name): + """Get the name of the categories for a given list name""" + categories = table_api_call( + instance=instance, + table="sys_choice", + params={"sysparm_query": f"name={list_name}^element=category", "sysparm_fields": "value"}, + )["result"] + + return categories diff --git a/src/browsergym/workarena/api/change_request.py b/src/browsergym/workarena/api/change_request.py new file mode 100644 index 0000000..9a8cbb2 --- /dev/null +++ b/src/browsergym/workarena/api/change_request.py @@ -0,0 +1,87 @@ +import faker +import numpy as np + +fake = faker.Faker() + +from datetime import datetime, timedelta + +from .category import get_categories +from .utils import table_api_call + +from ..instance import SNowInstance + + +def create_change_request( + instance: SNowInstance, + user_sys_id: str, + impact: int, + risk: int, + start_date: datetime = "", + end_date: datetime = "", + hashtag: str = "", + short_description: str = None, + random: np.random = None, +) -> list[str]: + """ + Create a change request + + Parameters: + ----------- + instance: SNowInstance + The instance to create the change request in + user_sys_id: str + The sys_id of the user to assign the problem to + impact: str + The impact of the change request; ranges from 1 (high) to 3 (low) + risk: str + The risk of the change request; ranges from 2 (high) to 4 (low) + start_date: datetime.datetime + The start date of the change request; empty if not set + end_date: datetime.datetime + The end date of the change request; empty if not set + hashtag: str + The name of the hashtag for the change request. If "", no hashtag will be added + short_description: str + The short description of the change request. If None, a random sentence will be generated + random: np.random + The random number generator + + Returns: + -------- + sys_id of the change request + number of the change request + + """ + if short_description is None: + short_description = fake.sentence(4) + categories = get_categories(instance=instance, list_name="change_request") + category = random.choice(categories) + + cfg = { + "reason": "broken", + "upon_reject": "cancel", + "type": "emergency", + "state": "-5", + "phase": "requested", + "impact": str(impact), + "active": "true", + "short_description": short_description + " " + hashtag, + "assigned_to": user_sys_id, + "start_date": str(start_date), + "end_date": str(end_date), + "upon_approval": "proceed", + "justification": fake.sentence(), + "implementation_plan": fake.sentence(), + "phase_state": "open", + "risk": str(risk), + "cab_required": "false", + "category": category, + } + result = table_api_call( + instance=instance, + table="change_request", + method="POST", + json=cfg, + )["result"] + + return result["sys_id"], result["number"] diff --git a/src/browsergym/workarena/api/computer_asset.py b/src/browsergym/workarena/api/computer_asset.py new file mode 100644 index 0000000..85188fb --- /dev/null +++ b/src/browsergym/workarena/api/computer_asset.py @@ -0,0 +1,90 @@ +import json +import numpy as np +import time + +from faker import Faker + +fake = Faker() + +from ..instance import SNowInstance +from .utils import table_api_call + + +def create_computer_asset( + instance: SNowInstance, + asset_tag: str, + warranty_expiration_date: str = None, + user_sys_id: str = None, + computer_model_info: dict = None, + random: np.random = None, +): + """Create a hardware asset -computer model- and assign it to a user + Args: + -------- + instance (SNowInstance): + The instance to create the hardware asset in + asset_tag (str): + The asset tag of the hardware asset + warranty_expiration_date (str): + The warranty expiration date of the hardware asset. If None, a random date is chosen + user_sys_id (str): + The sys_id of the user to assign the hardware asset to. If None, the hardware asset is not assigned to any user + computer_model_info (dict): + Contains the sys_id and short_description of the computer model to create the hardware asset with. + If None, a random computer model is chosen + random (np.random): + The random number generator + Returns: + -------- + sys_id (str): + The sys_id of the created hardware asset + computer_model (dict): + The computer model information + warranty_expiration_date (str): + The warranty expiration date of the hardware asset + """ + + if computer_model_info is None: + # Get the sys_id of the 'Computer' category + computer_model_sys_id = table_api_call( + instance=instance, + table="cmdb_model_category", + # The cmdb_model_category is the sys_id for the hardware category; computer in this case + params={ + "sysparm_query": f"name=Computer", + "sysparm_fields": "sys_id", + }, + )["result"][0]["sys_id"] + # Randomly choose a computer model if needed + computer_models = table_api_call( + instance=instance, + table="cmdb_model", + # The cmdb_model_category is the sys_id for the hardware category; + params={ + "sysparm_query": f"cmdb_model_category={computer_model_sys_id}", + "sysparm_fields": "sys_id,short_description", + }, + )["result"] + computer_model = random.choice(computer_models) + if warranty_expiration_date is None: + # Warranty expiration date is randomly selected between 1 year ago and 1 year from now + warranty_expiration_date = str(fake.date_between(start_date="-1y", end_date="+1y")) + + # Create hardware asset + hardware_result = table_api_call( + instance=instance, + table="alm_hardware", + data=json.dumps( + { + "assigned_to": user_sys_id, + "asset_tag": asset_tag, + "display_name": asset_tag + " - " + computer_model["short_description"], + "model": computer_model["sys_id"], + "model_category": computer_model_sys_id, + "warranty_expiration": warranty_expiration_date, + } + ), + method="POST", + )["result"] + + return hardware_result["sys_id"], computer_model, warranty_expiration_date diff --git a/src/browsergym/workarena/api/cost_center.py b/src/browsergym/workarena/api/cost_center.py new file mode 100644 index 0000000..bd9a48a --- /dev/null +++ b/src/browsergym/workarena/api/cost_center.py @@ -0,0 +1,19 @@ +import warnings +from faker import Faker + +fake = Faker() + +from .utils import table_api_call + +from ..instance import SNowInstance + + +def get_cost_center_sysid(instance, cost_center_name): + """Get the sys_id of a cost center by its name""" + sys_id = table_api_call( + instance=instance, + table="cmn_cost_center", + params={"sysparm_query": f"name={cost_center_name}", "sysparm_fields": "sys_id"}, + )["result"][0] + + return sys_id diff --git a/src/browsergym/workarena/api/expense_line.py b/src/browsergym/workarena/api/expense_line.py new file mode 100644 index 0000000..df11e3c --- /dev/null +++ b/src/browsergym/workarena/api/expense_line.py @@ -0,0 +1,89 @@ +import json +import time + +from faker import Faker + +fake = Faker() + +from .cost_center import get_cost_center_sysid +from .utils import table_api_call + +from ..instance import SNowInstance + + +def create_expense_line( + instance: SNowInstance, + amount: float, + number: str, + date: str, + short_description: str = None, + expense_hashtag: str = "", + task_sys_id: str = None, + cost_center_sys_id: str = None, + summary_type: str = "run_business", + user_sys_id: str = None, +): + """Create a hardware asset -computer model- and assign it to a user + Args: + -------- + instance (SNowInstance): + The instance to create the hardware asset in + amount (float): + The amount of the expense line + number (str): + The number of the expense line + date (str): + The date of the expense line + short_description (str): + The short description of the expense line; if None, a random one will be generated + expense_hashtag (str): + The hashtag of the expense line (added to the short description) + task_sys_id (str): + The sys id of the task to file the expense line under + cost_center_sys_id (str): + The sys id of the cost center to file the expense line under + summary_type (str): + The summary type of the expense line (choice of "run_business", "grow_business", "transform_business") + user_sys_id (str): + The sys_id of the user to assign the hardware asset to. If None, the hardware asset is not assigned to any user + Returns: + -------- + sys_id (str): + The sys_id of the created expense_line + expense_line_number (str): + The number of the created expense_line + """ + if cost_center_sys_id is None: + # sys_id of the engineering cost center + cost_center_sys_id = table_api_call( + instance=instance, + table="cmn_cost_center", + params={"sysparm_query": "name=Engineering"}, + )["result"][0]["sys_id"] + + if short_description is None: + short_description = fake.sentence(4) + + expense_cfg = { + "date": date, + "base_expense": "", + "short_description": short_description + " " + expense_hashtag, + "summary_type": summary_type, + "summary_type": "run_business", + "type": "one-time", + "number": f"{number}", + "task": f"{task_sys_id}", + "state": "processed", + "amount": f"{amount}", + "cost_center": f"{cost_center_sys_id}", + "user": f"{user_sys_id}", + } + + result = table_api_call( + instance=instance, + table="fm_expense_line", + json=expense_cfg, + method="POST", + )["result"] + + return result["sys_id"], result["number"] diff --git a/src/browsergym/workarena/api/incident.py b/src/browsergym/workarena/api/incident.py new file mode 100644 index 0000000..844ac22 --- /dev/null +++ b/src/browsergym/workarena/api/incident.py @@ -0,0 +1,45 @@ +from faker import Faker +from ..instance import SNowInstance + +fake = Faker() + +from .utils import table_api_call + + +def create_incident( + instance: SNowInstance, + incident_number: int, + caller_sys_id: str, + category: str, + impact: int, + urgency: int, + priority: int, + incident_hastag: str = None, + assigned_to: str = None, +): + incident_config = { + "task_effective_number": incident_number, + "number": incident_number, + "state": 2, + "knowledge": False, + "impact": impact, + "active": True, + "priority": priority, + "caller_id": caller_sys_id, + "short_description": incident_hastag if incident_hastag else " ".join(fake.words(5)), + "description": " ".join(fake.words(10)), + "incident_state": 2, + "urgency": urgency, + "severity": 3, + "category": category, + } + if assigned_to: + incident_config["assigned_to"] = assigned_to + + incident_response = table_api_call( + instance=instance, + table="incident", + json=incident_config, + method="POST", + )["result"] + return incident_response diff --git a/src/browsergym/workarena/api/knowledge.py b/src/browsergym/workarena/api/knowledge.py new file mode 100644 index 0000000..3b55e83 --- /dev/null +++ b/src/browsergym/workarena/api/knowledge.py @@ -0,0 +1,29 @@ +from ..instance import SNowInstance +from .utils import table_api_call + + +def give_kb_read_permissions(admin_instance, user_sys_id, user_name, kb_sys_id, kb_name): + # Need admin permissions to give KB permissions to the user + + # Create user criteria + user_criteria_data = { + "user": user_sys_id, + "name": f"{user_name} read KB", + "short_description": f"Let {user_name} read {kb_name}", + } + criteria_response = table_api_call( + instance=admin_instance, table="user_criteria", json=user_criteria_data, method="POST" + )["result"] + criteria_sys_id = criteria_response["sys_id"] + + # Add user criteria entry to allow users to access the ADHOC KB + kb_uc_can_read_mtom_data = { + "user_criteria": criteria_sys_id, + "kb_knowledge_base": kb_sys_id, + } + _ = table_api_call( + instance=admin_instance, + table="kb_uc_can_read_mtom", + json=kb_uc_can_read_mtom_data, + method="POST", + )["result"] diff --git a/src/browsergym/workarena/api/problem.py b/src/browsergym/workarena/api/problem.py new file mode 100644 index 0000000..146bb6d --- /dev/null +++ b/src/browsergym/workarena/api/problem.py @@ -0,0 +1,90 @@ +import faker + +fake = faker.Faker() + +from ..instance import SNowInstance +from .utils import table_api_call + + +def create_problem( + instance: SNowInstance, + priority: str, + user_sys_id: str, + problem_hashtag: str, + short_description: str = None, + return_number: bool = False, +) -> list[str]: + """ + Create a problem with a random cause, description, and short description. The problem is assigned to a user and + is created with a hashtag. + + Parameters: + ----------- + instance: SNowInstance + The instance to create the problem in + priority: str + The priority of the problem + user_sys_id: str + The sys_id of the user to assign the problem to + problem_hashtag: str + The name of the hashtag for the problem + short_description: str + The short description of the problem (optional). if not provided, a random one will be generated + return_number: bool + whether or not to return the problem number that was created + + Returns: + -------- + sys_id of the problem + problem_number (optional) + + """ + cause = fake.sentence() + description = fake.text() + if short_description is None: + short_description = fake.sentence(4) + + # Priority is a read-only field defined by a combo of impact and urgency. The mapping is as follows: + # https://docs.servicenow.com/bundle/washingtondc-it-service-management/page/product/problem-management/concept/prioritise-problems.html + priority_to_impact_and_urgency = { + 1: (1, 1), + 2: (1, 2), + 3: (1, 3), + 4: (2, 3), + 5: (3, 3), + } + + impact, urgency = priority_to_impact_and_urgency[priority] + + problem_cfg = { + "made_sla": True, + "upon_reject": "cancel", + "cause_notes": f"

{cause}

", + "fix_notes": " placeholder ", # placeholder value - will not work without a fix note + "knowledge": False, + "major_problem": False, + "impact": f"{impact}", + "active": False, + "sys_domain_path": "/", + "short_description": f"{short_description} {problem_hashtag}", + "known_error": False, + "description": f"{description}", + "sla_due": "2021-04-11 17:39:07", + "closed_at": "", + "resolution_code": "fix_applied", + "urgency": f"{urgency}", + "assigned_to": f"{user_sys_id}", + "active": True, + } + + result = table_api_call( + instance=instance, + table="problem", + json=problem_cfg, + method="POST", + )["result"] + + if return_number: + return result["sys_id"], result["number"] + + return result["sys_id"] diff --git a/src/browsergym/workarena/api/report.py b/src/browsergym/workarena/api/report.py new file mode 100644 index 0000000..edb06f9 --- /dev/null +++ b/src/browsergym/workarena/api/report.py @@ -0,0 +1,183 @@ +import numpy as np + +from ..instance import SNowInstance +from .utils import table_api_call + + +def create_report( + instance: SNowInstance, + table: str, + filter_hashtag: str, + field: str, + plot_title: str, + filter_field: str = "short_description", + random: np.random = None, +) -> list[str]: + """ + Create a report for for the given table using a filter (str added to the short description). + The report is created with a random color palette and colors and a random plot type (pie or bar). + + Parameters: + ----------- + instance: SNowInstance + The instance to create the category in + table: str + The name of the table used to make the plot + field: str + The field of the table used to make the plot + filter_hashtag: str + The name of the hashtag to filter the table with + plot_title: str + The title of the plot + + Returns: + -------- + sys_id: str; sys_id of the report created + plot_title: str; The title of the plot + + """ + # select a random color palette for the plot + color_palettes = table_api_call( + instance=instance, + table="pa_chart_color_schemes", + params={ + "sysparm_fields": "sys_id", + }, + method="GET", + )["result"] + + color_palette_sys_id = random.choice(color_palettes)["sys_id"] + + # Get available colors to eventually randomly select from them + colors = table_api_call( + instance=instance, + table="sys_report_color", + params={ + "sysparm_fields": "sys_id", + }, + method="GET", + )["result"] + + # Select a random plot type + plot_types = ["pie", "bar"] + plot_type = random.choice(plot_types) + + report_params = { + "show_data_label_position_middle": False, + "display_row_lines": False, + "is_published": False, + "chart_title_y_position": "0", + "gauge_autoscale": True, + "type": f"{plot_type}", + "formatting_configuration": { + "table": "incident", + "stringFormattingProperties": {}, + "durationFormattingProperties": {}, + "dateFormattingProperties": {}, + "numberFormattingProperties": {}, + }, + "apply_alias": False, + "chart_border_color": f"{random.choice(colors)['sys_id']}", # default: "65b30218a9fe3dba0120df8611520d97" + "custom_chart_title_position": False, + "other_threshold": "-2", + "y_axis_title_color": f"{random.choice(colors)['sys_id']}", # default: "65b30218a9fe3dba0120df8611520d97" + "show_legend_border": False, + "donut_width_percent": "30", + "y_axis_title_size": "12", + "legend_border_color": f"{random.choice(colors)['sys_id']}", # default: "65b30218a9fe3dba0120df8611520d97" + "y_axis_label_bold": False, + "chart_title_size": "16", + "x_axis_label_color": f"{random.choice(colors)['sys_id']}", # default: "65b30218a9fe3dba0120df8611520d97" + "active": True, + "source_type": "table", + "x_axis_title_bold": True, + "x_axis_opposite": False, + "chart_height": "450", + "legend_border_radius": "0", + "field": field, + "show_geographical_label": False, + "legend_horizontal_alignment": "center", + "interval": "year", + "show_zero": False, + "y_axis_grid_width": "1", + "chart_subtitle_size": "14", + "x_axis_display_grid": False, + "show_chart_total": False, + "chart_subtitle_style": "normal", + "chart_title_style": "normal", + "x_axis_grid_width": "1", + "x_axis_label_tilt": "0", + "show_chart_title": "report", + "title_vertical_alignment": "top", + "legend_border_width": "1", + "compute_percent": "aggregate", + "show_marker": False, + "sys_scope": "global", + "map": "93b8a3a2d7101200bd4a4ebfae61033a", + "use_color_heatmap": False, + "use_null_in_trend": False, + "y_axis_label_tilt": "0", + "table": table, + "legend_vertical_alignment": "bottom", + "x_axis_label_bold": False, + "no_bulk_migration": False, + "filter": f"{filter_field}LIKE{filter_hashtag}", + "display_column_lines": False, + "x_axis_allow_decimals": True, + "custom_chart_size": False, + "title_horizontal_alignment": "center", + "bar_unstack": False, + "y_axis_allow_decimals": True, + "chart_width": "600", + "x_axis_title_size": "12", + "y_axis_opposite": False, + "y_axis_title_bold": True, + "decimal_precision": "2", + "y_axis_label_size": "11", + "x_axis_grid_color": f"{random.choice(colors)['sys_id']}", + "legend_align_columns": True, + "field_list": "active,short_description,incident_state,business_duration,calendar_duration,description,caller_id,location,closed_by,impact,cmdb_ci,priority,assigned_to,activity_due,task_effective_number,company,escalation,sys_created_on,closed_at,child_incidents,state,assignment_group,category,number,business_stc", + "is_scheduled": False, + "others": True, + "x_axis_title_color": f"{random.choice(colors)['sys_id']}", # default: "65b30218a9fe3dba0120df8611520d97" + "aggregation_source": "no_override", + "chart_title_color": f"{random.choice(colors)['sys_id']}", # default: "65b30218a9fe3dba0120df8611520d97" + "y_axis_grid_dotted": False, + "chart_size": "large", + "legend_items_left_align": False, + "show_chart_data_label": False, + "y_axis_grid_color": f"{random.choice(colors)['sys_id']}", + "allow_data_label_overlap": False, + "chart_border_radius": "0", + "title": plot_title, + "exp_report_attrs": True, + "aggregate": "COUNT", + "y_axis_display_grid": True, + "score_color": f"{random.choice(colors)['sys_id']}", # default: "65b30218a9fe3dba0120df8611520d97" + "axis_max_color": f"{random.choice(colors)['sys_id']}", # default: "b0d449b3d7332100fa6c0c12ce610383" + "is_real_time": False, + "show_empty": False, + "direction": "minimize", + "display_grid": False, + "chart_border_width": "1", + "funnel_neck_percent": "30", + "show_legend": True, + "set_color": "one_color", + "color_palette": f"{color_palette_sys_id}", # default: "65b30218a9fe3dba0120df8611520d97" + "x_axis_label_size": "11", + "show_chart_border": False, + "x_axis_grid_dotted": False, + "chart_title_x_position": "0", + "pivot_expanded": True, + } + + result = table_api_call( + instance=instance, + table="sys_report", + # The cmdb_model_category is the sys_id for the hardware category; computer in this case + json=report_params, + method="POST", + wait_for_record=True, + )["result"] + + return result["sys_id"], plot_title diff --git a/src/browsergym/workarena/api/requested_items.py b/src/browsergym/workarena/api/requested_items.py new file mode 100644 index 0000000..2856557 --- /dev/null +++ b/src/browsergym/workarena/api/requested_items.py @@ -0,0 +1,63 @@ +import faker + +fake = faker.Faker() + +from ..instance import SNowInstance +from .utils import table_api_call + + +def create_requested_item( + instance: SNowInstance, + user_sys_id: str, + system_name: str, + quantity: int = 1, + short_description: str = None, +) -> list[str]: + """ + Create a requested item for a given user + + Parameters: + ----------- + instance: SNowInstance + The instance to create the requested item in + user_sys_id: str + The sys_id of the user to request the item for + system_name: str + The name of the system to request (e.g. "Developer Laptop (Mac)" ) + quantity: int + The quantity of the item to request + short_description: str + The short description of the item (optional). if not provided, a random one will be generated + + Returns: + -------- + sys_id, number of the requested item + + """ + if short_description is None: + short_description = fake.sentence(4) + + item_sys_id = table_api_call( + instance=instance, + table="sc_cat_item", + params={"sysparm_query": f"sys_name={system_name}", "sysparm_fields": "sys_id"}, + )["result"][0]["sys_id"] + + item_config = { + "requested_for": user_sys_id, + "state": "3", + "impact": "3", + "active": "true", + "priority": "4", + "short_description": short_description, + "urgency": "3", + "quantity": str(quantity), + "billable": "false", + "cat_item": item_sys_id, + } + + result = table_api_call( + instance=instance, table="sc_req_item", json=item_config, method="POST" + )["result"] + + return result["sys_id"], result["number"] diff --git a/src/browsergym/workarena/api/user.py b/src/browsergym/workarena/api/user.py index b666de8..3e966ce 100644 --- a/src/browsergym/workarena/api/user.py +++ b/src/browsergym/workarena/api/user.py @@ -1,5 +1,5 @@ -import random from faker import Faker +import numpy as np import time fake = Faker() @@ -14,7 +14,9 @@ def create_user( first_name: str = None, last_name: str = None, user_name: str = None, - admin=True, + return_full_response: bool = False, + user_roles: list[str] = ["admin"], + random: np.random = np.random, ) -> list[str]: """ Create a user with a random username and password with an admin role @@ -27,8 +29,8 @@ def create_user( The last name of the user, defaults to a random last name user_name: str The user name of the user, defaults to first_name.last_name - admin: bool - Whether to give the user admin permissions + user_roles: list[str] + The roles to assign to the user, defaults to ['admin'] Returns: -------- @@ -56,12 +58,12 @@ def create_user( user_name = user_response["user_name"] user_sys_id = user_response["sys_id"] - # Get admin role sys_id - if admin: + # Get role sys_id's + for role in user_roles: role_sys_id = table_api_call( instance=instance, table="sys_user_role", - params={"sysparm_query": "name=admin", "sysparm_fields": "sys_id"}, + params={"sysparm_query": f"name={role}", "sysparm_fields": "sys_id"}, method="GET", )["result"][0]["sys_id"] @@ -77,7 +79,8 @@ def create_user( set_user_preference( instance, "glide.ui.polaris.theme.variant", theme["style.sys_id"], user=user_sys_id ) - + if return_full_response: + return user_response return user_name, user_password, user_sys_id diff --git a/src/browsergym/workarena/api/utils.py b/src/browsergym/workarena/api/utils.py index dc6aac5..47a6b2a 100644 --- a/src/browsergym/workarena/api/utils.py +++ b/src/browsergym/workarena/api/utils.py @@ -3,6 +3,7 @@ from ..instance import SNowInstance from requests.exceptions import HTTPError +from time import sleep # ServiceNow API configuration SNOW_API_HEADERS = {"Content-Type": "application/json", "Accept": "application/json"} @@ -15,12 +16,17 @@ def table_api_call( params: dict = {}, json: dict = {}, method: str = "GET", + wait_for_record: bool = False, + max_retries: int = 5, + raise_on_wait_expired: bool = True, ) -> dict: """ Make a call to the ServiceNow Table API Parameters: ----------- + instance: SNowInstance + The ServiceNow instance to interact with table: str The name of the table to interact with data: dict @@ -31,6 +37,13 @@ def table_api_call( The JSON data to send with the request method: str The HTTP method to use (GET, POST, PUT, DELETE). + wait_for_record: bool + If True, will wait up to 2 seconds for the record to be present before returning + max_retries: int + The number of retries to attempt before failing + raise_on_wait_expired: bool + If True, will raise an exception if the record is not found after max_retries. + Otherwise, will return an empty result. Returns: -------- @@ -49,13 +62,44 @@ def table_api_call( params=params, json=json, ) + if method == "POST": + sys_id = response.json()["result"]["sys_id"] + data = {} + params = {"sysparm_query": f"sys_id={sys_id}"} - # Check for HTTP code 200 (fail otherwise) + # Check for HTTP success code (fail otherwise) response.raise_for_status() + record_exists = False + num_retries = 0 + if method == "POST" or wait_for_record: + while not record_exists: + sleep(0.5) + get_response = table_api_call( + instance=instance, + table=table, + params=params, + json=json, + data=data, + method="GET", + ) + record_exists = len(get_response["result"]) > 0 + num_retries += 1 + if num_retries > max_retries: + if raise_on_wait_expired: + raise HTTPError(f"Record not found after {max_retries} retries") + else: + return {"result": []} + if method == "GET": + response = get_response + if method != "DELETE": - # Decode the JSON response into a dictionary - return response.json() + # Decode the JSON response into a dictionary if necessary + # When using wait_for_record=True, the response is already a dict as it is a recursive call + if type(response) == dict: + return response + else: + return response.json() else: return response diff --git a/src/browsergym/workarena/config.py b/src/browsergym/workarena/config.py index 4cd87e2..a3b7e8b 100644 --- a/src/browsergym/workarena/config.py +++ b/src/browsergym/workarena/config.py @@ -66,6 +66,10 @@ # Knowledge base that is included with the benchmark KB_NAME = "General Knowledge" KB_FILEPATH = str(resources.files(data_files).joinpath("setup_files/knowledge/knowledge_base.json")) +PROTOCOL_KB_NAME = "Company Protocols" +PROTOCOL_KB_FILEPATH = str( + resources.files(data_files).joinpath("setup_files/knowledge/protocols.json") +) # Form tasks CREATE_CHANGE_REQUEST_CONFIG_PATH = str( @@ -165,12 +169,25 @@ "setup_files/lists/expected_change_request_list_columns.json" ) ) +EXPECTED_EXPENSE_LINE_COLUMNS_PATH = str( + resources.files(data_files).joinpath( + "setup_files/lists/expected_expense_line_list_columns.json" + ) +) EXPECTED_HARDWARE_COLUMNS_PATH = str( resources.files(data_files).joinpath("setup_files/lists/expected_hardware_list_columns.json") ) EXPECTED_INCIDENT_COLUMNS_PATH = str( resources.files(data_files).joinpath("setup_files/lists/expected_incident_list_columns.json") ) +EXPECTED_PROBLEM_COLUMNS_PATH = str( + resources.files(data_files).joinpath("setup_files/lists/expected_problem_list_columns.json") +) +EXPECTED_REQUESTED_ITEMS_COLUMNS_PATH = str( + resources.files(data_files).joinpath( + "setup_files/lists/expected_requested_items_list_columns.json" + ) +) EXPECTED_SERVICE_CATALOG_COLUMNS_PATH = str( resources.files(data_files).joinpath( "setup_files/lists/expected_service_catalog_list_columns.json" @@ -188,6 +205,7 @@ "setup_files/forms/expected_change_request_form_fields.json" ) ) + EXPECTED_HARDWARE_FORM_FIELDS_PATH = str( resources.files(data_files).joinpath("setup_files/forms/expected_hardware_form_fields.json") ) @@ -200,7 +218,9 @@ EXPECTED_USER_FORM_FIELDS_PATH = str( resources.files(data_files).joinpath("setup_files/forms/expected_user_form_fields.json") ) - +EXPECTED_REQUEST_ITEM_FORM_FIELDS_PATH = str( + resources.files(data_files).joinpath("setup_files/forms/expected_request_item_form_fields.json") +) # Report date filter patch flag REPORT_PATCH_FLAG = "WORKARENA_DATE_FILTER_PATCH" diff --git a/src/browsergym/workarena/data_files/setup_files/forms/expected_incident_form_fields.json b/src/browsergym/workarena/data_files/setup_files/forms/expected_incident_form_fields.json index e417829..af9444c 100644 --- a/src/browsergym/workarena/data_files/setup_files/forms/expected_incident_form_fields.json +++ b/src/browsergym/workarena/data_files/setup_files/forms/expected_incident_form_fields.json @@ -1 +1 @@ -["comments", "assigned_to", "assignment_group", "caller_id", "category", "caused_by", "rfc", "contact_type", "cmdb_ci", "description", "impact", "knowledge", "number", "hold_reason", "parent_incident", "priority", "problem_id", "close_code", "close_notes", "resolved_at", "resolved_by", "business_service", "service_offering", "short_description", "state", "subcategory", "route_reason", "universal_request", "urgency", "watch_list", "work_notes", "work_notes_list"] \ No newline at end of file +["comments", "assigned_to", "assignment_group", "caller_id", "category", "caused_by", "rfc", "contact_type", "cmdb_ci", "description", "impact", "knowledge", "number", "hold_reason", "parent_incident", "priority", "problem_id", "close_code", "close_notes", "resolved_at", "resolved_by", "business_service", "service_offering", "short_description", "state", "subcategory", "route_reason", "universal_request", "urgency", "watch_list", "work_notes", "work_notes_list"] diff --git a/src/browsergym/workarena/data_files/setup_files/forms/expected_request_item_form_fields.json b/src/browsergym/workarena/data_files/setup_files/forms/expected_request_item_form_fields.json new file mode 100644 index 0000000..bb1f82b --- /dev/null +++ b/src/browsergym/workarena/data_files/setup_files/forms/expected_request_item_form_fields.json @@ -0,0 +1 @@ +["number", "cat_item", "request", "requested_for", "due_date", "configuration_item", "watch_list", "opened_at", "opened_by", "stage", "state", "quantity", "estimated_delivery", "backordered", "order_guide", "comments"] diff --git a/src/browsergym/workarena/data_files/setup_files/knowledge/protocols.json b/src/browsergym/workarena/data_files/setup_files/knowledge/protocols.json new file mode 100644 index 0000000..6bde16b --- /dev/null +++ b/src/browsergym/workarena/data_files/setup_files/knowledge/protocols.json @@ -0,0 +1,46 @@ +[ + { + "item": "Onboarding a new user", + "article": "

Onboarding a new user

This document outlines the procedure for onboarding a new user within the company. Proper onboarding ensures that new users have the necessary tools and access to start their roles effectively and efficiently. Follow the steps below to complete the onboarding process.

Steps for Onboarding a New User

1. Create a User Account

  1. Access the user creation interface.
  2. Enter the required information for the new user.
  3. Save the new user account.

You can create a user account here.

2. Order a Laptop

  1. Access the service catalog.
  2. Order an Apple MacBook Pro 15\" laptop for the new user.
  3. Confirm the order details and submit the request.

You can order the laptop here.

3. Create a Hardware Asset

  1. Access the hardware asset management interface.
  2. Create a new hardware asset with the required configuration.
  3. Assign the newly created hardware asset to the user.

You can create and assign a hardware asset here.

Conclusion

Following these steps will ensure that new users are properly onboarded with the necessary equipment and access rights. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to streamline the user onboarding process, ensuring consistency and efficiency across the organization.

" + }, + { + "item": "Offboarding a user", + "article": "

Offboarding a user

Introduction

This document outlines the procedure for offboarding a user within the company. Proper offboarding ensures that company assets are secured and that the departing user's access is properly removed. Follow the steps below to complete the offboarding process.

Steps for Offboarding a User

1. Un-assign Hardware Assets

  1. Find the user's laptop by looking for the hardware asset assigned to them.
  2. Un-assign the laptop by erasing the \"Assigned to\" field.

You can manage hardware assets in the hardware asset list.

2. Delete the User Account

  1. Access the user management interface.
  2. Locate and delete the user account.

You can delete the user account here.

Conclusion

Following these steps will ensure that departing users are properly offboarded and that all company assets and access rights are appropriately managed. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to streamline the user offboarding process, ensuring consistency and security across the organization.

" + }, + { + "item": "Finding the warranty expiration for a user's laptop", + "article": "

Finding the Warranty Expiration for a User's Laptop

Introduction

This document outlines the procedure for finding the warranty expiration date of a user's laptop. Ensuring that the warranty information is up-to-date is crucial for managing hardware assets and planning for replacements or repairs.

Steps to Find the Warranty Expiration Date

1. Locate the Laptop Assigned to the User

  1. Find the laptop assigned to the user by looking at the \"Assigned to\" field in the hardware asset list.

2. Find the Laptop's Warranty Expiration Date

  1. Once the laptop is located, check the warranty expiration date on the asset details page.

Conclusion

Following these steps will help you accurately find the warranty expiration date of a user's laptop. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to ensure proper management of hardware warranties within the organization.

" + }, + { + "item": "Agent Workload Balancing", + "article": "

Agent Workload Balancing

Introduction

Agent Workload Balancing is a process designed to distribute work evenly among agents by re-assigning tasks. The problems to re-distribute all contain specific hashtags in their problem statements. By balancing the workload, we ensure that no single agent is overwhelmed while others have fewer tasks. All problems can be found in the problem list.

Steps for Agent Workload Balancing

1. Identify Busiest and Least Busy Users

  1. Among the problems with descriptions containing the required hashtag, identify the user with the most assigned problems and the user with the least assigned problems.
  2. This information can be found in the report named 'Problems with hashtag {hashtag_name}'.

You can access the list of reports here.

2. Find a Low Priority Problem

  1. Locate a problem with the lowest priority (priority=5) that includes the appropriate hashtag in the problem statement and is assigned to the busiest user.

You can filter the problem list to find such a problem.

3. Re-assign the Problem

  1. Re-assign the identified low priority problem to the least busy user.

Conclusion

Following these steps will help balance the workload among agents by re-assigning low priority problems from the busiest to the least busy user. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to ensure effective workload management among agents within the organization.

" + }, + { + "item": "Work Assignment: Assign Incidents to Relevant Agents", + "article": "

Work Assignment: Assign Incidents to Relevant Agents

Introduction

This document outlines the process for assigning incidents to relevant agents based on both the category and the priority of the incident. Proper assignment ensures that incidents are handled by the most appropriate agents, leading to quicker and more effective resolutions. Follow these steps to assign incidents correctly.

Steps for Assigning Incidents

1. Locate the Incident

  1. Go to the incidents page and search for the incident using the incident number that needs to be assigned.

You can access the incidents page here.

2. Identify the Incident Category and Priority

  1. Look at the category and priority of the incident. The category will be one of the following: 'Hardware', 'Software', 'Network', or 'Database'.
  2. Assign the incident to the category experts mentioned in the task description.

3. Assign the Incident

  1. If applicable, assign the incident to the relevant agent based on the priority of the incident. The priority levels are as follows:
    • Priority 1 - Critical: Assign to the 'expert' of the category.
    • Priority 3 - Moderate: Assign to the 'supporter' of the category.
    • Priority 5 - Planning: Assign to the 'planner' of the category.

Conclusion

By following these steps, you can ensure that incidents are assigned to the most qualified agents based on their category and priority. This approach leads to faster resolution times and improved efficiency in incident management. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to ensure proper assignment of incidents to relevant agents within the organization.

" + }, + { + "item": "Edit a knowledge article to manage incorrect information", + "article": "

Edit a Knowledge Article to Manage Incorrect Information

Introduction

This document outlines the procedure for editing a knowledge base article that contains incorrect, outdated, or obsolete information. Ensuring that all knowledge articles are accurate and up-to-date is crucial for maintaining the integrity and reliability of the knowledge base.

Steps to Edit a Knowledge Article

1. Search the Knowledge Base

  1. Search the knowledge base using the given query.

You can perform the search here.

2. Identify the Correct and Incorrect Articles

  1. Identify the articles that contain the correct and incorrect information based on what the information should be.

3. Mark the Incorrect Article

  1. Add a comment to the article to mark it as 'incorrect' and mention the 'number' of the correct article as per the task description.

4. Mark the Correct Article

  1. Add a comment to the article to label it as 'correct' and mention the 'number' of the incorrect article as per the task description to avoid having misguiding people.

Conclusion

Following these steps will help maintain the accuracy and relevance of the knowledge base by properly marking and referencing articles with correct and incorrect information. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to ensure effective management of information within the knowledge base.

" + }, + { + "item":"Problem List Cleanup", + "article": "

Problem List Cleanup

Introduction

Problem List Cleanup ensures that the problem list remains organized and free from redundancy. In the problem list, some problem's \"Problem statement\" field contain a hashtag so we know which problem group they belong to. The problem list cleanup involves identifying problems with duplicated problem statements and marking them as duplicates.

Steps for Problem List Cleanup

1. Identify Duplicated Problems

  1. Among the problems with problem statements containing the required hashtag, identify those with duplicated problem statements. This can be done by filtering the problem list .

2. Mark Duplicated Problems

  1. If two problems have the same priority, you can mark either one as a duplicate of the other. Otherwise, mark the lowest priority problem as a duplicate of the highest priority one to keep the highest priority problems open. A problem's priority can range from 1 (Critical) to 5 (Planning), with 1 being the highest priority.
  2. Marking a problem as duplicate is done by opening the problem record and Clicking \"Mark Duplicate\".

3. Additional Handling for Critical Priority Problems

  1. If you need to mark a priority 1 (Critical priority) problem as a duplicate of another, change the description of this problem to \"Duplicate\" to avoid further confusion.

Conclusion

Following these steps will help maintain a clean and efficient problem list by properly marking and managing duplicated problems. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to ensure effective problem management within the organization.

" + }, + { + "item": "Scheduling Your Change Requests", + "article": "

Scheduling Your Change Requests

Introduction

Scheduling change requests is a critical process in managing IT infrastructure and operations. This process involves organizing and prioritizing change requests to ensure they are implemented efficiently and with minimal disruption to ongoing operations. Change requests often include updates, fixes, or enhancements that need to be scheduled within specific time blocks. The following guidelines outline how to create a valid schedule for change requests. As a first step, we note that change requests are grouped by hashtags placed in their short description. In general, creating a schedule is done for change requests that share a given hashtag and by setting the \"Planned start date\" and \"Planned end date\" for these change requests. Make sure to leave empty the \"Actual start date\" and \"Actual end date\" as they should only be filled when the work has started. All change requests can be found here.

Scheduling Rules

1. Time Constraints

  1. All change requests must be scheduled within the allowed time frame. They should not start before the start of the schedule block and must end before it closes. Moreover, they should not overlap.

2. Impact Order

  1. Higher impact change requests must be scheduled before lower impact ones, ensuring the schedule prioritizes more significant changes.

3. Consecutive Requests

  1. Generally, there should be no more than one day between consecutive change requests.
  2. If the change requests are required to follow a \"tight\" schedule, there should be no more than one hour between consecutive requests.

4. Duration Compliance

  1. All change requests must respect the desired durations, which are determined by the risk level:
    • High risk: 3 days
    • Moderate risk: 2 days
    • Low risk: 1 day

Conclusion

Following these guidelines will help you create an efficient and effective schedule for your change requests, minimizing disruptions and prioritizing higher impact changes. For any issues or additional assistance, please contact the IT department.

This document serves as a guide to ensure proper scheduling and management of change requests within the organization.

" + }, + { + "item": "Managing Your Existing Expenses", + "article": "

Managing Your Existing Expenses

Introduction

Managing your existing expenses is a crucial aspect of maintaining financial accuracy and compliance within the company. This process involves reviewing expense lines to ensure no duplicates remain. Expense lines can be found here. Duplicate expenses can lead to inaccuracies in financial reporting and budget management, making it essential to handle them according to established guidelines. Expense lines normally contain specific hashtags so they can be grouped. The process of filtering the duplicates is done for specific hashtags.

Steps for Managing Expense Duplicates

1. Identify Duplicate Expenses

  1. Review expense lines with short descriptions containing the given hashtag.
  2. Identify expense lines that have the same short description, which are considered duplicates.

2. Apply Rules to Handle Duplicates

  1. Among duplicates, keep only those linked to tasks (e.g., Change Request, Incident) and delete the others.
  2. If all duplicates are linked to tasks, or none are linked to tasks, keep only the oldest one.
  3. If all duplicates have the same date, keep only the most expensive one.
  4. Finally, if all duplicates have the same cost, you can keep any of the expenses.

Conclusion

Following these guidelines will help you manage your existing expenses effectively, ensuring compliance with company policies and maintaining accurate financial records. For any issues or additional assistance, please contact the finance department.

This document serves as a guide to ensure proper management of expenses within the organization.

" + }, + { + "item": "Maximizing total investment return", + "article": "

Maximizing Total Investment Return

Introduction

Investment projects are filed as expense lines and are tagged with project IDs (hashtags) so they can be grouped. Expense lines can be found here. Maximizing total investment return involves reviewing these expense lines and selecting the ones that maximize revenue while fitting within the allowed budget. The investment's returns are specified in their short descriptions and their costs appear in their \"Amount\" field. Depending on the specifics of the task, you may be required to provide different types of information.

Steps for Maximizing Total Investment Return

1. Review Expense Lines

  1. Identify expense lines with short descriptions containing the given hashtag.
  2. Evaluate the returns based on the short description and costs based on the \"Amount\" field to determine their suitability based on the allowed budget.

2. Provide Required Information

Depending on the task requirements, you may need to provide one or more of the following:

  • Total Return of Selected Investments Only: If requested, provide only the total return of the selected investments in the chat.
  • Selected Investments Only: If requested, provide the values for the \"Number\" field of the selected investments in the chat.
  • Selected Investments and Total Return: If requested, provide the values for the \"Number\" field of the selected investments as well as their total return in the chat.
  • Investment Lines Cleanup: If requested, delete the investments that will not be kept so that only the selected investments remain.

Conclusion

Following these steps will help you allocate investments effectively, ensuring that the selected projects maximize revenue while adhering to the budget constraints. For any issues or additional assistance, please contact the finance department.

This document serves as a guide to ensure proper management and allocation of investments within the organization.

" + }, + { + "item": "Dashboard Retrieve Information and Perform Task", + "article": "

Dashboard Retrieve Information and Perform Task

Introduction

Dashboards consists of either bar plots or pie charts containing important information, generally as a pair of some text and numbers. You would be asked to retrieve some information from the chart, which might be retrieving values such as the maximum and minimum value or requiring standard mathematical calculations such as mean, median, and mode. You may also require to extract the text associated with these values, such as the name of an agent. Once they are retrieved, you would be requested to perform a task using this information. Please note that we round off everything to the next highest integer and treat it as the 'value' for the task. For calculations having more than one possible answer (such as 'mode' or 'most frequent' values), we consider the greatest number as the value or information.

Steps for performing dashboard retrieval tasks

1. Retrieve information from the dashboard

  1. Go to Reports > View/Run in the all menu. You can go to the page here as well.
  2. Use the hashtag given in the task description to filter out the required dashboard chart and navigate to it.
  3. Based on the task description, you will be given a 'value' field. This would mention if you need to fetch the minimum or maximum valued information or the mean, median, or the mode.
  4. In certain cases, the description will also mention if you need to perform the following task by considering 'greater than or equal to' or 'lesser than or equal to' values for the retrieved information.

This is the retrieved information from the dashboard.

2. Perform the task

After retrieving the information, you would be given a task to solve. Depending on the task requirements, you may need to perform one or more of the following:

  • Create form entries: Fill forms with the given information. Only fill the fields mentioned in the task description.
  • Filter lists: Filter the lists using the attribute mentioned in the task description and following the retrieved information.
  • Order items: You would need to go to the service catalog and order an item after making some calculations of the quantity as per the task description.

The following links may be helpful for achieving the above tasks:

Conclusion

Following these steps will help you achieve the task of retrieving information from a dashboard and using it to solve another task.

This document serves as a guide to ensure proper management and allocation of investments within the organization.

" + } +] diff --git a/src/browsergym/workarena/data_files/setup_files/knowledge/test.html b/src/browsergym/workarena/data_files/setup_files/knowledge/test.html new file mode 100644 index 0000000..4fa89da --- /dev/null +++ b/src/browsergym/workarena/data_files/setup_files/knowledge/test.html @@ -0,0 +1 @@ +

Dispatching problems

\n

Dispatching problems involves taking problems under a given category and re-assigning one low priority from the busiest to the least busy user.To do so, you need to:

\n
    \n
  • Find the user that has the most problems assigned to them and the one that has the least number of problems assigned to them. This is done by finding the information in the report for the given category, which is named 'Problems for category {category_name}'. The list of reports can be found here
    • \n
  • Find a lowest priority problem (priority=5) of the appropriate category assigned to the user. This can be done by filtering the problem list
    • \n
  • Assign this problem to the least busy user
\ No newline at end of file diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_asset_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_asset_list_columns.json index 3556ccd..4d93121 100644 --- a/src/browsergym/workarena/data_files/setup_files/lists/expected_asset_list_columns.json +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_asset_list_columns.json @@ -1,34 +1,12 @@ [ + "asset_tag", "model.display_name", "model_category", - "asset_tag", - "substatus", - "asset_function", - "install_status", "sys_class_name", "assigned_to", - "serial_number", "location", - "vendor", - "assigned", "company", - "cost", "department", - "display_name", - "model", - "sys_updated_by", - "sys_updated_on", - "sys_mod_count", - "retirement_date", - "retired", - "residual", - "quantity", - "purchase_date", - "po_number", - "order_date", - "delivery_date", - "managed_by", - "invoice_number", - "cost_center", + "install_status", "warranty_expiration" ] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_change_request_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_change_request_list_columns.json index c210bb2..b458d5f 100644 --- a/src/browsergym/workarena/data_files/setup_files/lists/expected_change_request_list_columns.json +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_change_request_list_columns.json @@ -1,48 +1,12 @@ [ "number", "short_description", - "chg_model", - "type", - "state", "risk", + "impact", + "priority", + "assigned_to", "start_date", "end_date", - "requested_by", - "assignment_group", - "assigned_to", - "sys_created_on", - "active", - "work_end", - "work_start", - "approval", - "approval_history", - "approval_set", - "backout_plan", - "category", - "change_plan", - "close_code", - "close_notes", - "closed_at", - "closed_by", - "cmdb_ci", - "conflict_last_run", - "conflict_status", - "contact_type", - "sys_created_by", - "delivery_plan", - "delivery_task", - "description", - "task_effective_number", - "escalation", - "impact", "implementation_plan", - "justification", - "knowledge", - "made_sla", - "opened_at", - "opened_by", - "phase", - "phase_state", - "priority", - "reassignment_count" + "approval" ] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_expense_line_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_expense_line_list_columns.json new file mode 100644 index 0000000..d8ea687 --- /dev/null +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_expense_line_list_columns.json @@ -0,0 +1,12 @@ +[ + "number", + "user", + "inherited", + "parent", + "date", + "short_description", + "source_id", + "amount", + "type", + "summary_type" +] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_hardware_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_hardware_list_columns.json index 7f67cfc..d49408c 100644 --- a/src/browsergym/workarena/data_files/setup_files/lists/expected_hardware_list_columns.json +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_hardware_list_columns.json @@ -1,53 +1,12 @@ [ "display_name", "model_category", - "asset_tag", "serial_number", "assigned_to", "company", + "cost_center", "install_status", - "substatus", - "cost", "ci", - "asset_function", - "acquisition_method", - "active_to", - "assigned", - "beneficiary", - "checked_in", - "checked_out", - "sys_class_name", - "comments", - "cost_center", - "sys_created_on", - "sys_created_by", - "department", - "due", - "due_in", - "eligible_for_refresh", - "expenditure_type", - "install_date", - "invoice_number", - "justification", - "lease_id", - "location", - "managed_by", - "model", - "delivery_date", - "order_date", - "owned_by", - "po_number", "purchase_date", - "quantity", - "request_line", - "retirement_date", - "skip_sync", - "stockroom", - "support_group", - "supported_by", - "sys_updated_on", - "sys_updated_by", - "sys_mod_count", - "vendor", "warranty_expiration" ] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_incident_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_incident_list_columns.json index 442e748..35faf06 100644 --- a/src/browsergym/workarena/data_files/setup_files/lists/expected_incident_list_columns.json +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_incident_list_columns.json @@ -2,27 +2,11 @@ "number", "caller_id", "category", - "cmdb_ci", "priority", + "impact", "state", "short_description", - "assignment_group", "assigned_to", - "child_incidents", - "active", - "activity_due", - "business_duration", - "business_stc", - "closed_at", - "closed_by", "company", - "sys_created_on", - "sys_created_on", - "description", - "calendar_duration", - "task_effective_number", - "escalation", - "impact", - "incident_state", - "location" + "sys_created_on" ] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_problem_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_problem_list_columns.json new file mode 100644 index 0000000..0dc851a --- /dev/null +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_problem_list_columns.json @@ -0,0 +1,12 @@ +[ + "number", + "short_description", + "state", + "priority", + "resolution_code", + "assigned_to", + "cmdb_ci", + "related_incidents", + "category", + "company" +] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_requested_items_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_requested_items_list_columns.json new file mode 100644 index 0000000..3e86879 --- /dev/null +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_requested_items_list_columns.json @@ -0,0 +1,12 @@ +[ + "number", + "sc_catalog", + "cat_item", + "short_description", + "approval", + "request", + "requested_for", + "opened_by", + "quantity", + "stage" +] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_service_catalog_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_service_catalog_list_columns.json index ef9d625..d547bbf 100644 --- a/src/browsergym/workarena/data_files/setup_files/lists/expected_service_catalog_list_columns.json +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_service_catalog_list_columns.json @@ -1,29 +1,12 @@ [ "name", "short_description", - "active", - "roles", "sc_catalogs", "category", - "price", "type", - "sys_updated_on", - "access_type", - "sys_scope", - "availability", + "price", "sys_class_name", - "mobile_picture_type", - "sys_created_on", - "sys_created_by", "delivery_time", - "sys_name", "description", - "delivery_plan", - "fulfillment_automation_level", - "flow_designer_flow", - "no_cart_v2", - "no_wishlist_v2", - "no_attachment_v2", - "hide_sp", - "no_quantity_v2" + "fulfillment_automation_level" ] diff --git a/src/browsergym/workarena/data_files/setup_files/lists/expected_user_list_columns.json b/src/browsergym/workarena/data_files/setup_files/lists/expected_user_list_columns.json index 0a2dd95..c6dc384 100644 --- a/src/browsergym/workarena/data_files/setup_files/lists/expected_user_list_columns.json +++ b/src/browsergym/workarena/data_files/setup_files/lists/expected_user_list_columns.json @@ -2,58 +2,11 @@ "user_name", "name", "email", - "active", - "sys_created_on", - "sys_updated_on", "avatar", - "building", - "phone", - "calendar_integration", - "city", - "sys_class_name", + "title", "company", - "cost_center", - "country", - "sys_created_by", - "date_format", - "default_perspective", "department", - "sys_domain", - "sys_domain_path", - "employee_number", - "enable_multifactor_authn", - "failed_attempts", - "first_name", - "gender", - "home_phone", - "internal_integration_user", - "ldap_server", - "preferred_language", - "last_login", - "last_login_time", - "last_name", "location", - "locked_out", - "manager", - "middle_name", - "mobile_phone", - "notification", - "user_password", - "password_needs_reset", - "photo", - "introduction", - "roles", - "schedule", - "source", - "state", - "street", - "sys_tags", - "time_format", "time_zone", - "title", - "sys_updated_by", - "sys_mod_count", - "vip", - "web_service_access_only", - "zip" -] \ No newline at end of file + "phone" +] diff --git a/src/browsergym/workarena/data_files/task_configs/all_menu.json b/src/browsergym/workarena/data_files/task_configs/all_menu.json index 9f9e2da..9ddca2e 100644 --- a/src/browsergym/workarena/data_files/task_configs/all_menu.json +++ b/src/browsergym/workarena/data_files/task_configs/all_menu.json @@ -4999,4 +4999,4 @@ "module": "Forms > Workspace View Rules", "url": "/now/nav/ui/classic/params/target/sysrule_view_workspace_list.do%3Fsysparm_view%3Dworkspace" } -] \ No newline at end of file +] diff --git a/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_minmax_task.json b/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_minmax_task.json index 0d2e53a..b76715a 100644 --- a/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_minmax_task.json +++ b/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_minmax_task.json @@ -1 +1 @@ -[{"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Short Customer Satisfaction Survey using Smiley Face", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Service Desk Satisfaction Survey", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'First Call Resolve' 60 Days", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'First Call Resolve' 60 Days", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Service Desk Satisfaction Survey", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Short Customer Satisfaction Survey using Smiley Face", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "max"}] +[{"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Closed Changes per Month", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Closed Changes per Month", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "min"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "max"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "max"}] diff --git a/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_value_task.json b/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_value_task.json index 480019f..43f35d8 100644 --- a/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_value_task.json +++ b/src/browsergym/workarena/data_files/task_configs/dashboard_retrieval_value_task.json @@ -1 +1 @@ -[{"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Confidential"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Abel Tuter"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Assessment assessor"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; percent; Conversation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Multiple Selection"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; percent; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; count; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; count; Fulfillment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Personally identifiable information"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; count; Conversation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'First Call Resolve' 60 Days", "chart_series": "", "question": "value; count; No"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Bess Marso"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Warranty"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Groups"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; percent; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; count; Happy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Maintenance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Adobe Systems"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_upgrade_history_log"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; percent; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Customer Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Visio"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; percent; Poor"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; percent; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Allie Pumphrey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Adela Cervantsz"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Allie Pumphrey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Insurance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; APC"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; David Loo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; count; Happy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alene Rabeck"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; percent; Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sh$sys_ux_lib_asset"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_ui_element"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; count; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; count; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Assessment: Scale and Template question"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; percent; request_approved"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_app_scan_payload"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; Don Goodliffe"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Survey : Basic Platform Based Test"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; count; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; AIX"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; count; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Solaris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Database"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Yes/No"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Service Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; count; request_approved"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Office Small Business Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Gateway"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'First Call Resolve' 60 Days", "chart_series": "", "question": "value; percent; No"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Short Customer Satisfaction Survey using Smiley Face", "question": "value; count; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; count; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Canon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Printer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2003 Standard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_update_version"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; percent; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Office Professional Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; count; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; count; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; percent; Happy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; count; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Service Desk Satisfaction Survey", "question": "value; percent; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; count; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Maintenance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_app_scan_payload"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Lease"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; percent; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Windows Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alejandra Prenatt"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; count; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; UPS"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; count; Fine"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Don Goodliffe"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Question Bank : Assessment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Helpdesk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_ux_lib_component"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; count; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_ux_lib_component"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; percent; Popup"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Fully automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; percent; Standard Laptop"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000 Advanced Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; percent; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; count; Average"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Linux Red Hat"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Service Desk Satisfaction Survey", "question": "value; count; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_dictionary"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Office Visio Professional 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; count; Apple iPad 3"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; count; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sh$sys_attachment_doc"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows XP"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Image Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_ux_lib_asset"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Likert Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000 Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Linux Red Hat"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; Natasha Ingram"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_ux_macroponent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_documentation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; count; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Iris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Office Visio Professional 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; percent; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Warranty"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; percent; Unhappy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Iris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_upgrade_history_log"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Survey : Basic Platform Based Test"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Office Small Business Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; percent; Unhappy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; count; Unhappy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; count; Average"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000 Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_attachment_doc"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Abel Tuter"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Computer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; APC"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; count; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_attachment_doc"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Yes/No"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alene Rabeck"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; N/A"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; count; Very Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; percent; Fine"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; count; Very Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; percent; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Service Desk Satisfaction Survey", "question": "value; percent; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; count; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; percent; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Numeric Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; percent; Very Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; count; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; count; Samsung Galaxy S7"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Windows Cluster Node"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; percent; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; David Miller"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'First Call Resolve' 60 Days", "chart_series": "", "question": "value; percent; Yes"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Short Customer Satisfaction Survey using Smiley Face", "question": "value; percent; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; percent; Fulfillment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Cyberpower"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Fully automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Lease"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Internal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Hewlett-Packard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; EBC Customer Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Service Desk Satisfaction Survey", "question": "value; count; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Sony"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Numeric Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; count; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Office Professional Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; EBC Customer Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Service"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'First Call Resolve' 60 Days", "chart_series": "", "question": "value; count; Yes"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; percent; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_schema_attribute_m2m"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Windows Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; percent; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Template"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Image Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_documentation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_metadata"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Computer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Date/Time"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sysevent0006"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_script_include"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; AT&T"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; Carol Coughlin"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; Fred Luddy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; count; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; count; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; AIX"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Choice"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Percentage"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; percent; Very Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sysevent0006"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; percent; Just Ok"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Change Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Knowledge Lab Session Feedback Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alfonso Griglen"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_dictionary"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_storage_alias"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Bess Marso"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; count; Standard Laptop"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; count; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; iBUYPOWER"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Unspecified"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows NT 4.0"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; UNIX Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; David Miller"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Joe Employee"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; percent; Average"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Canon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2003 Standard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Question Bank Survey For ATF"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; AT&T"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; count; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Number"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Internal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Question Bank Survey For ATF"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; percent; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Personally identifiable information"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Gateway"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; percent; Excellent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; count; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Assessment assessor"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Visual Studio 2005 Professional Edition - ENU"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; count; Popup"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Mac OS 10 (OS/X)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alfonso Griglen"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; Joe Employee"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Short Customer Satisfaction Survey using Smiley Face", "question": "value; percent; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Intel"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Web Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; percent; Happy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; NDA"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Customer Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; N/A"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Cyberpower"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Visio"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Semi-automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; percent; Samsung Galaxy S7"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; percent; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Choice"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; percent; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Email Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Helpdesk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Assessment: Scale and Template question"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Template"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_rollback_sequence"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sh$sys_ux_lib_asset"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_rollback_sequence"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; percent; Average"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Cruz Roudabush"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Surveys by Metric Type and State", "chart_series": "Short Customer Satisfaction Survey using Smiley Face", "question": "value; count; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; Fred Luddy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Kodak"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; count; Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_ux_lib_asset"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Kodak"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Likert Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Confidential"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; syslog0007"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Rick Berzle"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_metadata"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; count; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; percent; Fine"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Restricted"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; count; Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Percentage"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Manual"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; Rick Berzle"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Question Bank : Assessment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_upgrade_state"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; percent; Excellent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; percent; Average"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Email Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Multiple Selection"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Intel"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; percent; Apple iPad 3"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; count; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alejandra Prenatt"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Adobe Systems"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; count; Excellent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; count; Poor"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000 Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; percent; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Number"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Software"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000 Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Contextual search", "question": "value; percent; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Joe Employee"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; count; Window"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Service"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Insurance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; Natasha Ingram"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; count; Just Ok"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Software"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; String"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Sony"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Manual"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Date"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Restricted"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Date"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; percent; Virtual agent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_update_version"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; Amazon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Knowledge Lab Session Feedback Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; UPS"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Symantec"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; percent; Good"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Default", "question": "value; percent; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_upgrade_state"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; Rick Berzle"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Courteous' 90 Days", "chart_series": "", "question": "value; count; Average"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000 Advanced Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sh$sys_attachment_doc"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Adela Cervantsz"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; count; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Hewlett-Packard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; David Loo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Tech Competence' 60 Days", "chart_series": "", "question": "value; count; Excellent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Recent items", "question": "value; percent; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Printer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; percent; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; count; Carol Coughlin"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; count; Dept. Head Approval"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_storage_alias"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows NT 4.0"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows XP"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; syslog0007"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; count; Fine"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Service Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; Don Goodliffe"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_schema_attribute_m2m"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Symantec"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Rick Berzle"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Semi-automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; Amazon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Change Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_script_include"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Solaris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; count; sys_ux_macroponent"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Mac OS 10 (OS/X)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Unspecified"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Database"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; NDA"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; UNIX Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; count; Unhappy"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D6b706f448f231110953ddffc9071a4f3", "chart_title": "Top 20 Tables Size", "chart_series": "", "question": "value; percent; sys_ui_element"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; String"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Web Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Visual Studio 2005 Professional Edition - ENU"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Assignment Group Customer Satisfaction 90 Day Average", "chart_series": "", "question": "value; percent; Joe Employee"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; percent; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; iBUYPOWER"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Don Goodliffe"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Timely Response' 90 Days", "chart_series": "", "question": "value; count; Just Ok"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Cruz Roudabush"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Groups"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; percent; Window"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Popular items", "question": "value; percent; Default"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D2d297c880f1130101527008c07767e27", "chart_title": "Service Desk Survey 'Overall Experience' 60 Days", "chart_series": "", "question": "value; percent; Just Ok"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; percent; Dept. Head Approval"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent contextual search", "question": "value; percent; Service portal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog request channel analytics - last 12 months", "chart_series": "Virtual agent", "question": "value; percent; Mobile"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Windows Cluster Node"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Date/Time"}] +[{"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2021-01-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2018-09-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Choice"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Unspecified"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; percent; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alene Rabeck"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000 Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; percent; request_approved"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2019-07-29"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2018-09-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Linux Red Hat"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2019-07-22"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2015-08-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000 Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; EBC Customer Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; iBUYPOWER"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; percent; Dept. Head Approval"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Number"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; count; 3 - Moderate"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000 Advanced Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2023-12-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2023-12-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Question Bank : Assessment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Cyberpower"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Template"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2019-07-29"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; String"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2023-11-20"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; percent; 4 - Low"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Bess Marso"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2019-07-22"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2016-08-08"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Office Small Business Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2023-12-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; percent; 2 - High"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2015-11-02"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alene Rabeck"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Mac OS 10 (OS/X)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2023-12-18"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; percent; 1 - Critical"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2024-03-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Semi-automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; N/A"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; AIX"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; David Miller"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Service"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Adobe Systems"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows NT 4.0"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Rick Berzle"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Template"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; UNIX Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Yes/No"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Abel Tuter"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2021-01-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Percentage"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2018-10-15"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2016-08-08"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; count; 5 - Planning"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2023-11-06"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; percent; Conversation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2003 Standard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Allie Pumphrey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alfonso Griglen"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Email Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; David Miller"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2020-06-01"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Groups"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Unspecified"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2024-03-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2021-01-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; count; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; Amazon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; UPS"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Linux Red Hat"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000 Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alejandra Prenatt"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; count; 2 - High"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2023-11-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Lease"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; percent; 2 - High"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Intel"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Canon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Gateway"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Web Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Numeric Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; count; 3 - Moderate"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Cruz Roudabush"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Computer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2024-02-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; count; 4 - Low"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Software"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Iris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Survey : Basic Platform Based Test"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; APC"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Likert Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Bess Marso"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2023-11-13"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2019-07-29"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; percent; Cancelled"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; percent; 3 - Moderate"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2015-11-02"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; count; 1 - Critical"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; count; Dept. Head Approval"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; percent; 4 - Low"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Cruz Roudabush"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Gateway"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; percent; 2 - High"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Hewlett-Packard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2024-02-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2023-11-13"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; EBC Customer Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Helpdesk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Office Professional Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Service"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2023-11-13"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Intel"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Kodak"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2016-12-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2016-08-08"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Manual"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; percent; 5 - Planning"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Mac OS 10 (OS/X)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Kodak"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2018-09-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; percent; Apple iPad 3"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; count; 2 - High"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Personally identifiable information"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Lease"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; count; Window"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; percent; 4 - Low"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Assessment assessor"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Printer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Image Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Numeric Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Email Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; NDA"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; count; request_approved"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Insurance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Personally identifiable information"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2023-11-13"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2020-06-01"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2023-12-18"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Canon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2016-12-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Printer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Date"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2019-07-22"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Cyberpower"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2019-07-29"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Fully automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; count; Apple iPad 3"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Adobe Systems"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2023-12-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2023-12-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Joe Employee"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Restricted"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Office Professional Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2024-02-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2023-12-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; IBM"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Visual Studio 2005 Professional Edition - ENU"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2018-08-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Closed Changes per Month", "chart_series": "", "question": "value; count; Mar/2024"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Date"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Service Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2015-08-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Warranty"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Date/Time"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Iris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Confidential"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows XP"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; count; Popup"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2024-02-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; count; Alva Pennigton"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Groups"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Symantec"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Percentage"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Web Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Office Visio Professional 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2015-08-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2018-12-03"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; percent; 3 - Moderate"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Windows Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alfonso Griglen"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2023-11-20"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Politeness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2023-11-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2015-08-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; percent; 1 - Critical"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Assessment: Scale and Template question"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Multiple Selection"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2016-12-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; percent; 1 - Critical"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; percent; Internal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Alejandra Prenatt"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows XP"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; UPS"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Restricted"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; N/A"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Warranty"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2015-11-02"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Closed Changes per Month", "chart_series": "", "question": "value; percent; July/2015"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; count; Standard Laptop"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Solaris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Question Bank Survey For ATF"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Knowledge Lab Session Feedback Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Likert Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Responsiveness"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; AT&T"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2023-11-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Alejandro Mascall"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; count; Samsung Galaxy S7"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Office Visio Professional 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Fully automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2016-12-12"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Sony"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2018-08-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows NT 4.0"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Windows Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Change Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2023-11-06"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; percent; Window"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2003 Standard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Allie Pumphrey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Apple"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Insurance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Sony"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2015-11-02"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Closed Changes per Month", "chart_series": "", "question": "value; count; July/2015"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; AT&T"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; NDA"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; iBUYPOWER"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000502"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; percent; Question Bank : Assessment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000505"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2020-06-01"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2016-08-08"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2023-11-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2018-09-10"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2018-12-03"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2023-11-06"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Adela Cervantsz"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Date/Time"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Image Scale"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; count; 2 - High"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Helpdesk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Question Bank Survey For ATF"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2023-11-06"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Lenovo"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Survey : Basic Platform Based Test"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2023-12-18"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2023-12-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; count; Fulfillment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2020-06-01"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Customer Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Change Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items by Stages", "chart_series": "", "question": "value; percent; Fulfillment"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2023-12-18"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; percent; Microsoft Office Small Business Edition 2003"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; count; 1 - Critical"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Yes/No"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2021-01-11"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Open Changes - Grouped", "chart_series": "", "question": "value; count; 1 - Critical"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Computer"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; UNIX Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; count; 4 - Low"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Knowledge Lab Session Feedback Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0002002"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Software"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2018-08-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2024-03-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; percent; Standard Laptop"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Hewlett-Packard"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Visual Studio 2005 Professional Edition - ENU"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Assessment assessor"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; count; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; count; 4 - Low"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; percent; Popup"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; APC"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2023-12-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; count; ASG0000501"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2018-10-15"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000503"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D3d48f669538223008329ddeeff7b1253", "chart_title": "Open Problems - Grouped", "chart_series": "", "question": "value; percent; 5 - Planning"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Dell Inc."}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0002000"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2018-08-27"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Joe Employee"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; percent; 2018-12-03"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Assessment: Scale and Template question"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; percent; Amazon"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Microsoft"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0002001"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Windows Cluster Node"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000804"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Abel Tuter"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; count; Maintenance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item VA render type", "chart_series": "", "question": "value; count; Conversation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Open Incidents - Grouped", "chart_series": "", "question": "value; count; 5 - Planning"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Database"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; percent; Windows 2000 Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Asus"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2018-10-15"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Number"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; NPS category"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Rick Berzle"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2018-12-03"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2023-11-20"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D05b0a8b7c3123010a282a539e540dd69", "chart_title": "Closed Changes per Month", "chart_series": "", "question": "value; percent; Mar/2024"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2018-10-15"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; percent; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Open", "question": "value; count; 2023-11-20"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Microsoft Licenses", "chart_series": "", "question": "value; count; Microsoft Windows XP Professional"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; percent; ASG0000803"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Don Goodliffe"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; Multiple Selection"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; percent; Bow Ruggeri"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; count; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Complete", "question": "value; count; ASG0000801"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Don Goodliffe"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; Manual"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Confidential"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by Manufacturer", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Visio"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; percent; Other"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Service Request Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; percent; Visio"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D287d07d1ff3130106c1ef9a7cddcbd5d", "chart_title": "Open Request Items", "chart_series": "", "question": "value; percent; Samsung Galaxy S7"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; percent; String"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Type", "chart_series": "", "question": "value; percent; Maintenance"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; count; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Stakeholders per Category", "chart_series": "", "question": "value; count; Recommendation"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Database"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessments by State", "chart_series": "", "question": "value; count; Complete"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Solaris"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Category Result Ratings by Category", "chart_series": "", "question": "value; count; Service Desk Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Total Metrics by Metric Type", "chart_series": "", "question": "value; percent; Customer Satisfaction Survey"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; count; 2024-03-04"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D68ee1f30770230107384c087cc5a992e", "chart_title": "Contract Expenditure by Vendor", "chart_series": "", "question": "value; count; (empty)"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dc38ca3a273031010ae8dd21efaf6a747", "chart_title": "Breakdown of classified data", "chart_series": "", "question": "value; count; Internal"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000504"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Answered Questions by Assigned User", "chart_series": "", "question": "value; count; Adela Cervantsz"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D7ab78953eb32011008f2951ff15228e6", "chart_title": "Catalog item fulfillment automation coverage", "chart_series": "", "question": "value; percent; Semi-automated"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; Windows 2000 Advanced Server"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Computers by OS", "chart_series": "", "question": "value; count; AIX"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Unanswered Questions by Assigned User", "chart_series": "", "question": "value; percent; Billie Cowley"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D18b1f472533130104c90ddeeff7b12a6", "chart_title": "Incidents per week", "chart_series": "Closed", "question": "value; percent; 2019-07-22"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item by Manufacturer", "chart_series": "", "question": "value; count; Symantec"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Metrics by Data Type", "chart_series": "", "question": "value; count; Choice"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D812fa4400f1130101527008c07767e1a", "chart_title": "Assessment Instances by Assessment Group", "chart_series": "Cancelled", "question": "value; percent; ASG0000802"}, {"url": "/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3Dfa5fe3e1773130107384c087cc5a99d5", "chart_title": "Configuration Item Types", "chart_series": "", "question": "value; count; Windows Cluster Node"}] diff --git a/src/browsergym/workarena/data_files/task_configs/filter_service_catalog_item_list_task.json b/src/browsergym/workarena/data_files/task_configs/filter_service_catalog_item_list_task.json index aac5780..6c6c441 100644 --- a/src/browsergym/workarena/data_files/task_configs/filter_service_catalog_item_list_task.json +++ b/src/browsergym/workarena/data_files/task_configs/filter_service_catalog_item_list_task.json @@ -227953,4 +227953,4 @@ } } } -] \ No newline at end of file +] diff --git a/src/browsergym/workarena/data_files/task_configs/impersonation_users.json b/src/browsergym/workarena/data_files/task_configs/impersonation_users.json index a64fb19..c04a07f 100644 --- a/src/browsergym/workarena/data_files/task_configs/impersonation_users.json +++ b/src/browsergym/workarena/data_files/task_configs/impersonation_users.json @@ -599,4 +599,4 @@ "Warren Speach", "Wayne Webb", "Winnie Reich" -] \ No newline at end of file +] diff --git a/src/browsergym/workarena/data_files/task_configs/report_retrieval_minmax_task.json b/src/browsergym/workarena/data_files/task_configs/report_retrieval_minmax_task.json index dda0323..0d9f373 100644 --- a/src/browsergym/workarena/data_files/task_configs/report_retrieval_minmax_task.json +++ b/src/browsergym/workarena/data_files/task_configs/report_retrieval_minmax_task.json @@ -1 +1 @@ -[{"url": "/now/nav/ui/classic/params/target/sys_report_template.do%3Fsysparm_field%3Dsys_class_name%26sysparm_type%3Dpie%26sysparm_table%3Dalm_asset%26sysparm_from_list%3Dtrue%26sysparm_chart_size%3Dlarge%26sysparm_manual_labor%3Dtrue%26sysparm_query=sys_created_on 0 + ] + + return curriculum + + +@tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_attempt(5), reraise=True) +def validate_solution(env): + infos = [] + messages = [] + for p in env.context.pages: + reward, stop, message, info = env.task.validate(p, env.chat.messages) + + # If a terminal condition is encountered, return it. + if reward == 1 or (reward == 0 and stop): + return reward, stop, message, info + + infos.append(info) + messages.append(message) + + return reward, stop, ", ".join(messages), {"message": ", ".join([i["message"] for i in infos])} + + +def main(): + + # Initialize the argument parser + parser = argparse.ArgumentParser( + description="Get annotator info and log path from command line arguments." + ) + + # Define the command line arguments + parser.add_argument("--email", type=str, required=True, help="Email of the annotator") + parser.add_argument( + "--curriculum", + type=str, + required=True, + help='Path to the curriculum file (optional: use "random" for a random one)', + ) + parser.add_argument( + "--log", + type=str, + required=False, + default="human_eval_log.json", + help="Path to the log file", + ) + parser.add_argument("--reset-log", action="store_true", help="Reset the log file") + + # Parse the arguments + args = parser.parse_args() + + annotator_info = {"email": args.email} + logging.info(f"Annotator info: {annotator_info}") + + # Reset the log file if requested + logging.info(f"Log file: {args.log}") + if args.reset_log: + logging.info("Resetting log file") + json.dump([], open(args.log, "w")) + + # Loop over the curriculum + curriculum = load_curriculum(args.curriculum) + logging.info(f"Starting evaluation for {len(curriculum)} tasks") + for i, task_info in enumerate(curriculum): + + if task_already_evaluated(args.log, annotator_info, task_info): + logging.info(f"Task {task_info} already evaluated. Skipping.") + continue + + # Setup the environment + logging.info(f"Setting up environment for task {task_info}") + env = setup_environment(task_info) + + # Game loop + logging.info(f"Starting evaluation for task {task_info}") + start_time = time() + end = False + success = False + prev_chat_len = len(env.chat.messages) + while True: + human_console_set_progress_status( + f"Task {i + 1} / {len(curriculum)} --- Elapsed: {round(time() - start_time, 2)} sec.", + env.context, + ) + + # Event: Human marked task as infeasible + infeasible, infeasible_reason = infeasible_flag_activated(env.context) + if infeasible and not any([m["role"] == "infeasible" for m in env.chat.messages]): + logging.info(f"Human marked task as infeasible. Reason: {infeasible_reason}") + human_console_set_status("Task marked as infeasible.", env.context) + env.chat.messages.append({"role": "infeasible", "message": infeasible_reason}) + # TODO: There is a small glitch where if the user changes their message after, + # the new infeasible message will be saved instead of the initial one that + # was added to the chat messages. We can't stop after infeasible has been + # declared. + + # Event: Validation is required + if validation_flag_activated(env.context) or len(env.chat.messages) != prev_chat_len: + human_console_set_status("Validation in progress...", env.context) + + # Patch all chat messages + for m in env.chat.messages: + if not m.get("patched", False): + if m["role"] == "user": + m["role"] = "assistant" + elif m["role"] == "assistant": + m["role"] = "user" + m["patched"] = True + + reward, stop, message, info = validate_solution(env) + logging.info(f"Validation: {info} -- reward: {reward} -- stop: {stop}") + + if reward == 1: + human_console_set_status("Success!", env.context) + end = True + success = True + else: + if not end: # If we're not already stopping for another reason + if stop: + human_console_set_status( + "Task not completed. Stop required.", env.context + ) + end = True + success = False + else: + human_console_set_status("Task not completed. Keep going.", env.context) + + prev_chat_len = len(env.chat.messages) + reset_validation_flag(env.context) + + # Event: Human abandoned task + if abandon_flag_activated(env.context): + end = True + success = False + human_console_set_status("Task abandoned by human.", env.context) + + # Event: Task is finished + if end: + log_result( + path=args.log, + annotator_info=annotator_info, + task_info=task_info, + metrics={ + "duration": time() - start_time, + "success": success, + "infeasible": infeasible_reason if infeasible else None, + "abandoned": abandon_flag_activated(env.context), + "chat_messages": env.chat.messages, + }, + ) + sleep(3) # Sleep so human has time to read status before it closes + break + + sleep(0.1) + + human_console_set_status("Cleaning environment. This may take a while...", env.context) + env.close() + logging.info(f"Finished evaluation for task {task_info}") + + +if __name__ == "__main__": + main() diff --git a/src/browsergym/workarena/install.py b/src/browsergym/workarena/install.py index 5b83f9e..d9c3faf 100644 --- a/src/browsergym/workarena/install.py +++ b/src/browsergym/workarena/install.py @@ -17,11 +17,16 @@ # for knowledge base setup KB_FILEPATH, KB_NAME, + PROTOCOL_KB_FILEPATH, + PROTOCOL_KB_NAME, # For list setup EXPECTED_ASSET_LIST_COLUMNS_PATH, EXPECTED_CHANGE_REQUEST_COLUMNS_PATH, + EXPECTED_EXPENSE_LINE_COLUMNS_PATH, EXPECTED_HARDWARE_COLUMNS_PATH, EXPECTED_INCIDENT_COLUMNS_PATH, + EXPECTED_PROBLEM_COLUMNS_PATH, + EXPECTED_REQUESTED_ITEMS_COLUMNS_PATH, EXPECTED_SERVICE_CATALOG_COLUMNS_PATH, EXPECTED_USER_COLUMNS_PATH, # for form setup @@ -29,6 +34,7 @@ EXPECTED_HARDWARE_FORM_FIELDS_PATH, EXPECTED_INCIDENT_FORM_FIELDS_PATH, EXPECTED_PROBLEM_FORM_FIELDS_PATH, + EXPECTED_REQUEST_ITEM_FORM_FIELDS_PATH, EXPECTED_USER_FORM_FIELDS_PATH, # Patch flag for reports REPORT_PATCH_FLAG, @@ -269,7 +275,11 @@ def delete_knowledge_base(instance: SNowInstance, kb_id: str, kb_name: str): def create_knowledge_base( - instance: SNowInstance, kb_name: str, kb_data: dict, disable_commenting: bool = True + instance: SNowInstance, + kb_name: str, + kb_data: dict, + disable_commenting: bool = True, + add_article_name: bool = False, ): """ Create knowledge base and upload all articles. @@ -283,6 +293,9 @@ def create_knowledge_base( The knowledge base data to upload disable_commenting: bool Whether to disable commenting on the knowledge base + add_article_name: bool + Whether to add the article name to the article text. If False, the articles will be named "Article " + Otherwise, we will extract the article title from the 'item' field in the JSON file. """ logging.info(f"Installing knowledge base {kb_name}...") @@ -311,7 +324,10 @@ def create_knowledge_base( for i, kb_entry in enumerate(kb_data): logging.info(f"... Knowledge Base {kb_name} uploading article {i + 1}/{len(kb_data)}") article = kb_entry["article"] - + if add_article_name: + short_description = kb_entry["item"] + else: + short_description = f"Article {i + 1}" # Plant a new article in kb_knowledge table table_api_call( instance, @@ -319,7 +335,7 @@ def create_knowledge_base( method="POST", data=json.dumps( { - "short_description": f"Article {i + 1}", + "short_description": short_description, "sys_class_name": "kb_knowledge", "text": article, "article_type": "text", @@ -337,11 +353,12 @@ def setup_knowledge_bases(): """ # Get the ServiceNow instance instance = SNowInstance() - # Mapping between knowledge base name and filepath + whether or not to disable comments + # Mapping between knowledge base name and filepath + whether or not to disable comments + whether or not to add article name knowledge_bases = { - KB_NAME: (KB_FILEPATH, True), + KB_NAME: (KB_FILEPATH, True, False), + PROTOCOL_KB_NAME: (PROTOCOL_KB_FILEPATH, True, True), } - for kb_name, (kb_filepath, disable_commenting) in knowledge_bases.items(): + for kb_name, (kb_filepath, disable_commenting, add_article_name) in knowledge_bases.items(): # Load the knowledge base with open(kb_filepath, "r") as f: kb_data = json.load(f) @@ -365,6 +382,7 @@ def setup_knowledge_bases(): kb_name=kb_name, kb_data=kb_data, disable_commenting=disable_commenting, + add_article_name=add_article_name, ) # Confirm that the knowledge base was installed correctly @@ -570,10 +588,22 @@ def setup_list_columns(): "url": "/now/nav/ui/classic/params/target/incident_list.do", "expected_columns_path": EXPECTED_INCIDENT_COLUMNS_PATH, }, + "problem": { + "url": "/now/nav/ui/classic/params/target/problem_list.do", + "expected_columns_path": EXPECTED_PROBLEM_COLUMNS_PATH, + }, "sys_user": { "url": "/now/nav/ui/classic/params/target/sys_user_list.do", "expected_columns_path": EXPECTED_USER_COLUMNS_PATH, }, + "sc_req_item": { + "url": "/now/nav/ui/classic/params/target/sc_req_item_list.do", + "expected_columns_path": EXPECTED_REQUESTED_ITEMS_COLUMNS_PATH, + }, + "fm_expense_line": { + "url": "/now/nav/ui/classic/params/target/fm_expense_line_list.do", + "expected_columns_path": EXPECTED_EXPENSE_LINE_COLUMNS_PATH, + }, "sc_cat_item": { "url": "/now/nav/ui/classic/params/target/sc_cat_item_list.do", "expected_columns_path": EXPECTED_SERVICE_CATALOG_COLUMNS_PATH, @@ -680,6 +710,10 @@ def setup_form_fields(): "expected_fields_path": EXPECTED_USER_FORM_FIELDS_PATH, "url": "/now/nav/ui/classic/params/target/sys_user.do", }, + "create_request_item": { + "expected_fields_path": EXPECTED_REQUEST_ITEM_FORM_FIELDS_PATH, + "url": "/now/nav/ui/classic/params/target/sc_req_item.do", + }, } logging.info("... Creating a new user account to validate form fields") @@ -872,6 +906,17 @@ def wipe_system_admin_preferences(): ) +def is_report_filter_using_time(filter): + """ + Heuristic to check if a report is filtering based on time + + This aims to detect the use of functions like "gs.endOfToday()". To avoid hardcoding all of them, + we simply check for the use of keywords. Our filter is definitely too wide, but that's ok. + + """ + return "javascript:gs." in filter or "@ago" in filter + + def patch_report_filters(): """ Add filters to reports to make sure they stay frozen in time and don't show new data @@ -880,8 +925,6 @@ def patch_report_filters(): """ logging.info("Patching reports with date filter...") - cutoff_date = REPORT_DATE_FILTER - instance = SNowInstance() # Get all reports that are not already patched @@ -893,22 +936,35 @@ def patch_report_filters(): }, )["result"] - incompatible_reports = [] for report in reports: # Find all sys_created_on columns of this record. Some have many. sys_created_on_cols = [ c for c in table_column_info(instance, report["table"]).keys() if "sys_created_on" in c ] - try: # XXX: We purposely do not support reports with multiple filter conditions for simplicity if len(sys_created_on_cols) == 0 or "^NQ" in report["filter"]: - raise NotImplementedError() + logging.info(f"Discarding report {report['title']} {report['sys_id']}...") + raise NotImplementedError() # Mark for deletion + + if not is_report_filter_using_time(report["filter"]): + # That's a report we want to keep (use date cutoff filter) + filter_date = REPORT_DATE_FILTER + logging.info( + f"Keeping report {report['title']} {report['sys_id']} (columns: {sys_created_on_cols})..." + ) + else: + # XXX: We do not support reports with filters that rely on time (e.g., last 10 days) because + # there are not stable. In this case, we don't delete them but add a filter to make + # them empty. They will be shown as "No data available". + logging.info( + f"Disabling report {report['title']} {report['sys_id']} because it uses time filters..." + ) + filter_date = "1900-01-01" - # Add the filter filter = "".join( [ - f"^{col} 0 and not report["filter"].startswith("^") else "") @@ -921,16 +977,21 @@ def patch_report_filters(): "description": report["description"] + " " + REPORT_PATCH_FLAG, }, ) - logging.info( - f"Patched report {report['title']} {report['sys_id']} (columns: {sys_created_on_cols})..." - ) + logging.info(f"... done") except (NotImplementedError, HTTPError): # HTTPError occurs when some reports simply cannot be patched because they are critical and protected - incompatible_reports.append(report["sys_id"]) - logging.info( - f"Did not patch report {report['title']} {report['title']} (columns: {sys_created_on_cols})..." - ) + logging.info(f"...failed to patch report. Attempting delete...") + + # Delete the report if it cannot be patched + # This might fail sometimes, but it's the best we can do. + try: + table_api_call( + instance=instance, table=f"sys_report/{report['sys_id']}", method="DELETE" + ) + logging.info(f"...... deleted.") + except: + logging.error(f"...... could not delete.") @tenacity.retry( diff --git a/src/browsergym/workarena/tasks/base.py b/src/browsergym/workarena/tasks/base.py index 5b623e1..34a8c1e 100644 --- a/src/browsergym/workarena/tasks/base.py +++ b/src/browsergym/workarena/tasks/base.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from copy import deepcopy -from typing import Dict, List, Optional, Tuple +from typing import List, Optional, Tuple from uuid import uuid4 from urllib import parse @@ -18,7 +18,7 @@ from ..api.user import create_user from ..api.utils import table_api_call from ..config import SNOW_BROWSER_TIMEOUT, SNOW_JS_UTILS_FILEPATH -from ..utils import impersonate_user, url_login +from ..utils import url_login from ..instance import SNowInstance @@ -34,7 +34,8 @@ def __init__( start_rel_url: str, instance: SNowInstance = None, final_rel_url: Optional[str] = None, - username: Optional[str] = "admin", + user_roles: List[str] = ["admin"], + has_description: bool = False, ) -> None: """ Initialize the task @@ -45,10 +46,14 @@ def __init__( Random seed instance: SNowInstance The ServiceNow instance in which the task will be performed - start_url: str + start_rel_url: str The URL for the starting page of the task - final_url: str (optional) + final_rel_url: str (optional) The URL for the final page of the task (default: uses the value of base_url) + user_roles: list[str] + The roles to assign to the user (default: ["admin"]) + has_description: bool + Whether the task has a description in L3 compositional tasks """ super().__init__(seed) @@ -67,6 +72,16 @@ def __init__( self.final_url = self.start_url self.final_url_ = parse.urlparse(self.final_url) + # Set the task's unique ID + self.unique_id = str(uuid4()) + # Flag to ensure the task is setup only once + self.task_is_setup = False + self.delete_user_on_teardown = False + self.user_roles = user_roles + self.has_description = ( + has_description # Whether the task has a description in L3 compositional tasks + ) + def cheat(self, page: playwright.sync_api.Page, chat_messages: list[str]) -> None: # Don't call super cheat function because it's not implemented at the base level logging.debug("Cheat is solving the task") @@ -102,6 +117,8 @@ def setup(self, page: playwright.sync_api.Page, do_start=True) -> tuple[str, dic """ logging.debug("Setting up the base task") + if self.task_is_setup: + raise ValueError("The task is already setup") # Keep the page for client-side validation self.page = page @@ -109,6 +126,15 @@ def setup(self, page: playwright.sync_api.Page, do_start=True) -> tuple[str, dic # Set the page timeout page.set_default_timeout(SNOW_BROWSER_TIMEOUT) + # Create a new user to run the task if this is the starting task + if do_start: + self._base_initial_instance = self.instance + self._base_user_name, self._base_user_password, self._base_user_sysid = create_user( + instance=self.instance, user_roles=self.user_roles, random=self.random + ) + self.instance = deepcopy(self.instance) + self.instance.snow_credentials = (self._base_user_name, self._base_user_password) + self.delete_user_on_teardown = True # Set the task's unique ID self.unique_id = str(uuid4()) @@ -116,26 +142,26 @@ def setup(self, page: playwright.sync_api.Page, do_start=True) -> tuple[str, dic goal, info = self.setup_goal(page=page) # Load a few utility functions for init scripts - page.add_init_script(path=SNOW_JS_UTILS_FILEPATH) + page.context.add_init_script(path=SNOW_JS_UTILS_FILEPATH) # Add the initialization scripts to the page context for script in self.get_init_scripts(): page.context.add_init_script(script) - # Create a new user to run the task - self._base_initial_instance = self.instance - self._base_user_name, self._base_user_password, self._base_user_sysid = create_user( - self.instance - ) - self.instance = deepcopy(self.instance) - self.instance.snow_credentials = (self._base_user_name, self._base_user_password) - # Start the task if requested if do_start: self.start(page) + self.task_is_setup = True + return goal, info + def create_user(self, first_name: str = None, last_name: str = None): + """ + Create a user in the ServiceNow instance + + """ + @abstractmethod def setup_goal(self, page: playwright.sync_api.Page) -> tuple[str, dict]: """ @@ -157,11 +183,20 @@ def start(self, page: playwright.sync_api.Page) -> None: page.goto(self.start_url) def teardown(self) -> None: + """ + Clean up after the task + + Notes: + ------ + This method should not make assumptions on the state of the page (e.g., a specific URL). + + """ logging.debug("Tearing down the task") - # Delete the user - table_api_call( - instance=self._base_initial_instance, - table=f"sys_user/{self._base_user_sysid}", - method="DELETE", - ) + if self.delete_user_on_teardown: + # Delete the user + table_api_call( + instance=self._base_initial_instance, + table=f"sys_user/{self._base_user_sysid}", + method="DELETE", + ) diff --git a/src/browsergym/workarena/tasks/comp_building_block.py b/src/browsergym/workarena/tasks/comp_building_block.py new file mode 100644 index 0000000..fafd4f6 --- /dev/null +++ b/src/browsergym/workarena/tasks/comp_building_block.py @@ -0,0 +1,4 @@ +class CompositionalBuildingBlockTask: + """Base class for compositional building block tasks. Used to exclude these tasks from the list of tasks that are tested like atomic tasks""" + + pass diff --git a/src/browsergym/workarena/tasks/compositional/__init__.py b/src/browsergym/workarena/tasks/compositional/__init__.py new file mode 100644 index 0000000..708c5ef --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/__init__.py @@ -0,0 +1,76 @@ +from .utils.curriculum import AGENT_CURRICULUM, HUMAN_CURRICULUM + +ALL_COMPOSITIONAL_TASKS = [] + +for category, items in AGENT_CURRICULUM.items(): + category_tasks = [] + for task in items["buckets"]: + category_tasks += task + ALL_COMPOSITIONAL_TASKS += category_tasks + + +def specialize_task_class_to_level(task_cls, level): + """ + Function to hardcode the level for the tasks + """ + new_name = f"{task_cls.__name__}L{level}" + patched_cls = f""" +class {new_name}(task_cls): + def __init__(self, **kwargs): + super().__init__(level={level}, **kwargs) +""" + # Dictionary to capture local variables defined by exec + local_vars = {"task_cls": task_cls} + exec(patched_cls, globals(), local_vars) + return local_vars[new_name] + + +ALL_COMPOSITIONAL_TASKS_L2 = [ + specialize_task_class_to_level(task, level=2) for task in ALL_COMPOSITIONAL_TASKS +] +ALL_COMPOSITIONAL_TASKS_L3 = [ + specialize_task_class_to_level(task, level=3) for task in ALL_COMPOSITIONAL_TASKS +] + + +AGENT_CURRICULUM_L2 = dict() +AGENT_CURRICULUM_L3 = dict() + +for category, items in AGENT_CURRICULUM.items(): + AGENT_CURRICULUM_L2[category] = { + "buckets": [ + [specialize_task_class_to_level(task, level=2) for task in task_set] + for task_set in items["buckets"] + ], + "num_seeds": items["num_seeds"], + "weights": items["weights"], + } + AGENT_CURRICULUM_L3[category] = { + "buckets": [ + [specialize_task_class_to_level(task, level=3) for task in task_set] + for task_set in items["buckets"] + ], + "num_seeds": items["num_seeds"], + "weights": items["weights"], + } + +HUMAN_CURRICULUM_L2 = dict() +HUMAN_CURRICULUM_L3 = dict() + +for category, items in HUMAN_CURRICULUM.items(): + HUMAN_CURRICULUM_L2[category] = { + "buckets": [ + [specialize_task_class_to_level(task, level=2) for task in task_set] + for task_set in items["buckets"] + ], + "num_seeds": items["num_seeds"], + "weights": items["weights"], + } + HUMAN_CURRICULUM_L3[category] = { + "buckets": [ + [specialize_task_class_to_level(task, level=3) for task in task_set] + for task_set in items["buckets"] + ], + "num_seeds": items["num_seeds"], + "weights": items["weights"], + } diff --git a/src/browsergym/workarena/tasks/compositional/base.py b/src/browsergym/workarena/tasks/compositional/base.py new file mode 100644 index 0000000..452567e --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/base.py @@ -0,0 +1,364 @@ +import json +import time +import warnings + +from typing import List, Tuple +from playwright.sync_api._generated import Page + +from browsergym.workarena.config import PROTOCOL_KB_FILEPATH + +from .update_task import UpdatePrivateTask + +from ..base import AbstractServiceNowTask +from ..navigation import AllMenuTask + +from ...instance import SNowInstance + + +class CompositionalTask(AbstractServiceNowTask): + # Final private task instructions + final_private_task_instructions = 'Don\'t forget to mark this task as "Closed - complete" once successfully completed. If the task appears infeasible, mark the task as "Closed - skipped" .' + + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + start_rel_url: str = "/now/nav/ui/home", + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + protocol_name: str = "", + user_roles: List[str] = ["admin"], + ) -> None: + """ + Create a compositional task with specific subtasks + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + start_rel_url: str + The relative URL to start the task from. + fixed_config: list[AbstractServiceNowTask] + A list of subtasks. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + protocol_name: str + The name of the protocol to follow to complete the task; only used for level 3 tasks. + user_roles: list[str] + The roles to assign to the user (default: ["admin"]) + """ + super().__init__( + seed=seed, instance=instance, start_rel_url=start_rel_url, user_roles=user_roles + ) + # Set the task as completed in L3 + self.set_private_task_as_completed = True + self.seed = seed + + self.fixed_config = fixed_config + self.protocol_name = protocol_name + self.task_description = "" + self.short_description = "" + + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + if self.level == 2: + start_rel_url = "/now/nav/ui/home" + else: + self.private_task_id = "PTSK" + str(id(self) % (10**8)).zfill(8) + self.sys_id = None + start_rel_url = "" # For level 3 tasks, the start URL depends on the sys ID of the private task created for it + + def __len__(self) -> int: + return len(self.subtasks) + + def setup_goal( + self, + page: Page, + config: list[AbstractServiceNowTask], + build_pretty_print_description: bool = True, + ) -> tuple[str, str, dict]: + super().setup_goal(page=page) + # Index to keep track of the task we are currently validating + self.valid_index = 0 + + # Setup all the subtasks + self.subtasks = [] + self.subgoals = [] + for task in config: + if ( + self.level == 2 and not task.used_in_level_2 + ): # Skip tasks that are not used in level 2; e.g. navigate to the company protocol + continue + self.subtasks.append(task) + self.subgoals.append(self.subtasks[-1].setup(page=page, do_start=False)[0]) + + if self.level == 3: + if build_pretty_print_description: + self._build_pretty_printed_description(config) + level_3_final_tasks = [ + # Navigate to the My Work task list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Service Desk", + "module": "My Work", + "url": "/now/nav/ui/classic/params/target/task_list.do%3Fsysparm_userpref_module%3D1523b8d4c611227b00be8216ec331b9a%26sysparm_query%3Dactive%253Dtrue%255Eassigned_to%253Djavascript%253AgetMyAssignments%2528%2529%255Estate%2521%253D-5%255EEQ", + }, + is_validated=False, + used_in_level_2=False, + ), + # Close the private task + UpdatePrivateTask( + instance=self.instance, + fixed_config={ + "task_description": self.task_description, + "short_description": self.short_description, + }, + set_as_completed=self.set_private_task_as_completed, + is_validated=True, + used_in_level_2=False, + ), + ] + self.subtasks.extend(level_3_final_tasks) + # Set identical user credentials for all subtasks + for task in self.subtasks: + task._base_initial_instance = self.instance + task._base_user_name, task._base_user_password, task._base_user_sysid = ( + self._base_user_name, + self._base_user_password, + self._base_user_sysid, + ) + task.instance = self.instance + task.instance.snow_credentials = (self._base_user_name, self._base_user_password) + + # Finish the setup with the L3-specific tasks + for task in self.subtasks[-2:]: + task.setup(page=page, do_start=False) + # The sys ID of the private task is the sys ID of the last task in the list + self.sys_id = level_3_final_tasks[-1].sys_id + + self.start_url = ( + self.instance.snow_url + + f"/now/nav/ui/classic/params/target/vtb_task.do%3Fsys_id%3D{self.sys_id}" + ) + + # For level 2, include all substeps in the goal + # For level 3, the goal is already set in the private task + if self.level == 2: + task_intro = self.short_description + "\n" + # Get the protocol to follow for the task and pre-pend it to the goal + goal = task_intro + goal += " \n Concretely, you need to complete the following steps:" + + # In some cases, more than one subtasks with identical subgoals are present and the duplicated tasks have empty goals + # These multiple tasks are used to provide a complete cheat for the tasks like ManageChangeRequestScheduleTask subclasses + # To avoid having empty steps in the enumeration, we check if the goal is empty and skip if it is + i = 1 + for subgoal in self.subgoals: + if not subgoal: + continue + goal += f"\n{i}. {subgoal}" + i += 1 + + elif self.level == 3: + goal = f"Please complete the following task." + + return goal, {} + + def _get_config(self) -> list[AbstractServiceNowTask]: + """ + Get a configuration for a given compositional task, in the form of a list subtasks. + """ + raise NotImplementedError("This method should be implemented in a subclass") + + def cheat(self, page: Page, chat_messages: list[str], subtask_idx: int) -> None: + """ + Solve the a subtask of the task + + Parameters: + ---------- + page: Page + The page to solve the task on + chat_messages: list[str] + The list of messages in the chat + subtask_idx: int + The index of the subtask to solve. + + Note: + ----- + * We proceed separately for each subtask since this enables validation of each subtask separately. + This is useful for certifying the feasibility of tasks in the benchmark. Otherwise, cheat would + bring us to the final state of the task, which would make it impossible to validate subtasks. + * Use len(self) to get the number of subtasks in the task. + + """ + super().cheat(page, chat_messages) + self.subtasks[subtask_idx].cheat(page, chat_messages) + + def _build_pretty_printed_description(self, config: list[AbstractServiceNowTask]) -> str: + """ + Get the task information for the private task description; used for level 3 tasks. + Args: + config: list[AbstractServiceNowTask] + The list of subtasks in the task + """ + for subtask in config: + if subtask.is_validated or subtask.has_description: + self.task_description += subtask.get_pretty_printed_description() + self.task_description += "\n" + self.task_description += self.final_private_task_instructions + + return self.task_description + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + super().validate(page, chat_messages) + + # Initialize the index of the first subtask that requires validation + while ( + self.valid_index < len(self.subtasks) + and not self.subtasks[self.valid_index].is_validated + ): + self.valid_index += 1 + + if self.valid_index == len(self.subtasks): + return ( + 1, + True, + "Nice work, thank you!", + {"message": "Task completed successfully."}, + ) + # Validate the current subtask + subtask = self.subtasks[self.valid_index] + reward, stop, info, message = subtask.validate(page, chat_messages) + + # If the subtask is valid + if reward >= 1.0: + # ... override the info and message to avoid success messages from the subtask + info = message["message"] = ( + f"Step {self.valid_index + 1} has been completed successfully." + ) + # ... this is a subtask, so we don't want to stop + stop = False + # ... increment index to flag this one as solved + self.valid_index += 1 + + # If the subtask is not valid + else: + # ... contextualize the info and message per subtask + info = f"Step {self.valid_index + 1}: " + info + message["message"] = f"Step {self.valid_index + 1}: " + message.get("message", "") + + # Check if all subtasks are solved + if self.valid_index == len(self.subtasks): + return ( + 1, + True, + "Nice work, thank you!", + {"message": "Task completed successfully."}, + ) + + return 0, stop, info, message + + def teardown(self) -> None: + # XXX: In base.py we define the teardown method as being independent of the + # current state of the page. This means that we can just call all the + # subtasks' teardown methods. + for task in self.subtasks: + task.teardown() + super().teardown() + + +class InfeasibleCompositionalTask(CompositionalTask): + """ + Base class for infeasible tasks. + + Args: + -------- + infeasible_reason (List[str]): + The reason why the task is infeasible. If a task is infeasible, the validation will look for one of the reasons in the chat messages. + set by children classes. + """ + + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + start_rel_url: str = "/now/nav/ui/home", + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + protocol_name: str = "", + user_roles: List[str] = ["admin"], + infeasible_reasons: List[str] = [""], + provide_reason: bool = False, + **kwargs, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + start_rel_url=start_rel_url, + fixed_config=fixed_config, + level=level, + protocol_name=protocol_name, + user_roles=user_roles, + ) + # Set the private task as closed-skipped in L3 tasks + self.set_private_task_as_completed = False + self.provide_reason = provide_reason + self.infeasible_reasons = infeasible_reasons + self.infeasible_reason_found = ( + False # flag to check if the infeasible reason was found in the chat messages + ) + self.__dict__.update(kwargs) + + def cheat(self, page: Page, chat_messages: list[str], subtask_idx: int) -> None: + """Add the infeasible reasons to the chat messages""" + # Index of the infeasible task that requires sending a message in the chat for the cheat + # for L2 tasks, the last task is the infeasible one, whereas for L3 tasks, third from last is infeasible + cheat_index = len(self.subtasks) - 1 if self.level == 2 else len(self.subtasks) - 3 + # We expect the last task of the trajectory to be infeasible. Therefore, we do all but the last cheat step + if subtask_idx != cheat_index: + super().cheat(page, chat_messages, subtask_idx) + else: + message = ", ".join(self.infeasible_reasons) + chat_messages.append({"role": "infeasible", "message": message}) + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + """Check if the infeasible reasons are in the chat messages""" + if chat_messages and chat_messages[-1]["role"] == "infeasible": + answer = chat_messages[-1]["message"].lower() + else: + return ( + 0, + False, + "", + {"message": "The assistant did not provide an answer."}, + ) + if self.provide_reason and answer == "": + return ( + 0, + False, + "", + {"message": "The assistant did not provide a reason for the infeasibility."}, + ) + if not self.infeasible_reason_found: + for reason in self.infeasible_reasons: + if reason.lower() in answer: + self.infeasible_reason_found = True + break + if not self.infeasible_reason_found: + return ( + 0, + False, + "", + {"message": "The assistant did not provide the correct answer."}, + ) + + return super().validate(page, chat_messages) + + +class HumanEvalTask: + """Base class to label tasks suitable for human evaluation.""" + + pass diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_base.py b/src/browsergym/workarena/tasks/compositional/dash_do_base.py new file mode 100644 index 0000000..30d2951 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_base.py @@ -0,0 +1,1366 @@ +""" +Dashboard retrieval and do action comp tasks +""" + +import json +from functools import partial +import random +import numpy as np +from typing import List + +from faker import Faker + +fake = Faker() + +from playwright.sync_api._generated import Page + +from .base import CompositionalTask, InfeasibleCompositionalTask, HumanEvalTask +from .utils.infeasible_configs import get_infeasible_service_catalog_config +from ..base import AbstractServiceNowTask +from ..knowledge import KnowledgeBaseSearchTask + +from ...api.incident import create_incident +from ...api.report import create_report +from ...api.user import create_user +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.service_catalog import META_CONFIGS + + +class DashboardRetrieveAndDoTask(CompositionalTask, HumanEvalTask): + def __init__( + self, + instance: SNowInstance = None, + dashboard_class: AbstractServiceNowTask = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + dashboard_config: dict = None, + level: int = 2, + ) -> None: + """ + Generic task to perform a dashboard retrieval and perform a task. + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of tuples, each containing a subtask + dashboard_config: dict + Configuration to use for the dashboard task. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. Provided by the child class. + short_description: str + A short description of the task to be completed. Provided by the child class. + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + ) + self.used_in_level_2 = self.level == 2 + self.dashboard_config = dashboard_config + self.task_description = None + self.short_description = None + self.dashboard_class = dashboard_class + self.protocol_name = "Dashboard Retrieve Information and Perform Task" + self.description_mapping = { + "max": self.random.choice(["maximum", "highest", "greatest"]), + "min": self.random.choice(["minimum", "lowest", "least"]), + "mean": self.random.choice(["mean", "average"]), + "median": "median", + "mode": "mode (most frequent)", + } + + def create_report(self) -> None: + """ + Create task relevant dashboard report + """ + raise NotImplementedError + + def set_compositional_task(self) -> None: + """ + Create and return the compositional task + """ + raise NotImplementedError + + def get_compositional_task(self) -> list[AbstractServiceNowTask]: + """ + Return the compositional task + """ + return self.compositional_task + + def _get_config(self) -> list[AbstractServiceNowTask]: + + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": f"Can you find the '{self.protocol_name}' Protocol in the Knowledge Base?", + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + + dashboard_retrieval_subtask = [ + # Navigate to the reports list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Reports", + "module": "Administration > All", + "url": "/now/nav/ui/classic/params/target/sys_report_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Find the user with the desired config + self.dashboard_class( + instance=self.instance, + seed=None, + fixed_config=self.dashboard_config, + is_validated=False, + used_in_level_2=True, + ), + ] + + config = ( + navigate_to_protocol_subtask + + dashboard_retrieval_subtask + + self.get_compositional_task() + ) + return config + + def teardown(self) -> None: + return super().teardown() + + +class DashboardRetrieveAndDoInfeasibleTask(InfeasibleCompositionalTask): + def __init__( + self, + instance: SNowInstance = None, + dashboard_class: AbstractServiceNowTask = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + dashboard_config: dict = None, + level: int = 2, + ) -> None: + """ + Generic task to perform a dashboard retrieval and perform a task. + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of tuples, each containing a subtask + dashboard_config: dict + Configuration to use for the dashboard task. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. Provided by the child class. + short_description: str + A short description of the task to be completed. Provided by the child class. + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + ) + self.used_in_level_2 = self.level == 2 + self.dashboard_config = dashboard_config + self.task_description = None + self.short_description = None + self.dashboard_class = dashboard_class + self.protocol_name = "Dashboard Retrieve Information and Perform Task" + self.description_mapping = { + "max": self.random.choice(["maximum", "highest", "most"]), + "min": self.random.choice(["minimum", "lowest", "least"]), + "mean": self.random.choice(["mean", "average"]), + "median": "median", + "mode": "mode (most frequent)", + } + + def create_report(self) -> None: + """ + Create task relevant dashboard report + """ + raise NotImplementedError + + def set_compositional_task(self) -> None: + """ + Create and return the compositional task + """ + raise NotImplementedError + + def get_compositional_task(self) -> list[AbstractServiceNowTask]: + """ + Return the compositional task + """ + return self.compositional_task + + def _get_config(self) -> list[AbstractServiceNowTask]: + + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + has_description=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": f"Can you find the '{self.protocol_name}' Protocol in the Knowledge Base?", + "value": "", + }, + is_validated=False, + used_in_level_2=False, + has_description=False, + ), + ] + + dashboard_retrieval_subtask = [ + # Navigate to the reports list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Reports", + "module": "Administration > All", + "url": "/now/nav/ui/classic/params/target/sys_report_list.do", + }, + is_validated=False, + used_in_level_2=True, + has_description=False, + ), + # Find the user with the desired config + self.dashboard_class( + instance=self.instance, + seed=None, + fixed_config=self.dashboard_config, + is_validated=False, + used_in_level_2=True, + ), + ] + + config = ( + navigate_to_protocol_subtask + + dashboard_retrieval_subtask + + self.get_compositional_task() + ) + return config + + def teardown(self) -> None: + return super().teardown() + + +class DashboardRetrieveIncidentAndDoTask(DashboardRetrieveAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_incidents_per_agent: int = 4, + min_incidents_per_agent: int = 1, + num_agents: int = 4, + question: str = "", + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.incident_hashtag = ( + f"#INC{str(id(self) % (10**8)).zfill(9)}" # identifier to select problems + ) + self.chart_title = f"Incidents with hashtag {self.incident_hashtag}" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + dashboard_config={ + "url": "/now/nav/ui/classic/params/target/sys_report", + "chart_title": self.chart_title, + "question": question, + "chart_series": "", + }, + level=level, + dashboard_class=dashboard_class, + ) + self.question = question + self.max_incidents_per_agent = max_incidents_per_agent + self.min_incidents_per_agent = min_incidents_per_agent + self.num_agents = num_agents + if (self.max_incidents_per_agent - self.min_incidents_per_agent) < 2 or self.num_agents < 2: + raise Exception( + "The difference between maximum incidents and minimum incidents should be at least two. The number of agents should also be at least 2." + ) + self.task_description = f"You have to retrieve some information from a dashboard chart based on the description below. The chart presents the number of 'incidents' assigned to different agents. After retrieving the information, you will be asked to use it to complete a task.\n \n" + self.task_description += f"Title of the report: {self.incident_hashtag}\n\n" + if self.level == 3: + self.task_description += f"Referring to the company protocol '{self.protocol_name}' (located in the 'Company Protocols' knowledge base), complete the dashboard retrieval task.\n\n" + self.short_description = ( + f"Retrieve information from the chart with title {self.incident_hashtag} and perform the mentioned task." + + "\n For calculations, please round off to the next highest integer if required. If the required calculation has multiple possible answers (for example, 'mode' or 'most frequently' occuring value), please consider the highest value.\n\n" + ) + + def create_report( + self, + user_roles=["itil"], + ) -> None: + self.agents = {} + self.agent_sysids = [] + for _ in range(self.num_agents): + agent_response = create_user( + instance=self.instance, + first_name=f"{fake.first_name()}-{fake.first_name()}", + last_name=f"{fake.last_name()}-{fake.last_name()}", + return_full_response=True, + user_roles=user_roles, + ) + self.agents[agent_response["sys_id"]] = agent_response + self.agent_sysids.append(agent_response["sys_id"]) + + highest_agent = self.agent_sysids[ + -1 + ] # Choose last agent as the agent with maximum incidents + self.agents[highest_agent]["num_incidents"] = self.max_incidents_per_agent + self.agents[highest_agent]["incident_configs"] = [] + + lowest_agent = self.agent_sysids[ + 0 + ] # Choose first agent as the agent with minimum incidents + self.agents[lowest_agent]["num_incidents"] = self.min_incidents_per_agent + self.agents[lowest_agent]["incident_configs"] = [] + + for agent_sysid in self.agent_sysids[1:-1]: + self.agents[agent_sysid]["num_incidents"] = self.random.randint( + self.min_incidents_per_agent + 1, self.max_incidents_per_agent - 1 + ) + self.agents[agent_sysid]["incident_configs"] = [] + + number_assignments = sum([agent["num_incidents"] for agent in self.agents.values()]) + + all_existing_incidents = table_api_call( + instance=self.instance, table="incident", method="GET" + )["result"] + self.all_incident_numbers = [incident["number"] for incident in all_existing_incidents] + + self.new_incident_numbers = [] + for _ in range(number_assignments): + incident_number = ( + self.prefix + + str(id(self) % (10**8)).zfill(8)[:4] + + str(self.random.randint(1000, 9999)) + ) + while ( + incident_number in self.all_incident_numbers + or incident_number in self.new_incident_numbers + ): + incident_number = ( + self.prefix + + str(id(self) % (10**8)).zfill(8)[:4] + + str(random.randint(1000, 9999)) + ) + self.new_incident_numbers.append(incident_number) + + incident_number_idx = 0 + for agent, agent_attributes in self.agents.items(): + for _ in range(agent_attributes["num_incidents"]): + incident_response = create_incident( + instance=self.instance, + incident_number=self.new_incident_numbers[incident_number_idx], + caller_sys_id=self._base_user_sysid, + category="software", + priority=4, + impact=2, # priority is calculated as some combination of impact and urgency + urgency=3, + incident_hastag=self.incident_hashtag, + assigned_to=agent, + ) + self.agents[agent]["incident_configs"].append(incident_response) + incident_number_idx += 1 + + self.report_sys_id, _ = create_report( + instance=self.instance, + table="incident", + filter_hashtag=self.incident_hashtag, + field="assigned_to", + plot_title=self.chart_title, + random=self.random, + ) + + def get_agent_values(self, attribute_name, filter_than) -> list[str]: + agent_values = [] + agent_value_sysids = [] + agent_incidents = [ + agent_attributes["num_incidents"] for agent_attributes in self.agents.values() + ] + + if self.question == "max": + agent_value_sysids.append(self.agents[self.agent_sysids[-1]]["sys_id"]) + if attribute_name == "assigned_to": + agent_full_name = ( + self.agents[self.agent_sysids[-1]]["first_name"] + + " " + + self.agents[self.agent_sysids[-1]]["last_name"] + ) + agent_values.append(agent_full_name) + elif attribute_name == "first_name": + agent_first_name = self.agents[self.agent_sysids[-1]]["first_name"] + agent_values.append(agent_first_name) + else: + raise Exception("Filter column not supported.") + elif self.question == "min": + agent_value_sysids.append(self.agents[self.agent_sysids[0]]["sys_id"]) + if attribute_name == "assigned_to": + agent_full_name = ( + self.agents[self.agent_sysids[0]]["first_name"] + + " " + + self.agents[self.agent_sysids[0]]["last_name"] + ) + agent_values.append(agent_full_name) + elif attribute_name == "first_name": + agent_first_name = self.agents[self.agent_sysids[0]]["first_name"] + agent_values.append(agent_first_name) + else: + raise Exception("Filter column not supported.") + elif self.question == "mean" or self.question == "median" or self.question == "mode": + if self.question == "mean": + mean_incidents = np.mean(agent_incidents) + incidents_count = int(np.ceil(mean_incidents)) + elif self.question == "median": + incidents_count = int(np.ceil(np.median(agent_incidents))) + elif self.question == "mode": + # We select the maximum value if there are two or more modes + frequencies = {} + for count in agent_incidents: + if count not in frequencies: + frequencies[count] = 1 + else: + frequencies[count] += 1 + sorted_frequencies = { + count: frequency + for count, frequency in sorted( + frequencies.items(), key=lambda item: item[1], reverse=True + ) + } + max_frequency = list(sorted_frequencies.values())[0] + max_frequencies = [ + count + for count, frequency in sorted_frequencies.items() + if frequency == max_frequency + ] + incidents_count = int(max(max_frequencies)) + + for agent_sysid, agent_attributes in self.agents.items(): + if ( + filter_than == "greater" + and agent_attributes["num_incidents"] >= incidents_count + ) or ( + filter_than == "lesser" and agent_attributes["num_incidents"] <= incidents_count + ): + agent_value_sysids.append(agent_sysid) + if attribute_name == "assigned_to": + agent_full_name = ( + agent_attributes["first_name"] + " " + agent_attributes["last_name"] + ) + agent_values.append(agent_full_name) + + elif attribute_name == "first_name": + agent_first_name = agent_attributes["first_name"] + agent_values.append(agent_first_name) + else: + raise Exception("Filter column not supported.") + else: + raise Exception("Unsopprted question type.") + + return agent_values, agent_value_sysids + + def set_compositional_task(self) -> None: + raise NotImplementedError + + def teardown(self) -> None: + # Delete the report + db_delete_from_table( + instance=self.instance, + table="sys_report", + sys_id=self.report_sys_id, + ) + # Delete the incidents and users + for agent_sys_id in self.agents: + for incident in self.agents[agent_sys_id]["incident_configs"]: + db_delete_from_table( + instance=self.instance, + table="incident", + sys_id=incident["sys_id"], + ) + db_delete_from_table( + instance=self.instance, + table="sys_user", + sys_id=agent_sys_id, + ) + return super().teardown() + + +class DashboardRetrieveIncidentAndDoInfeasibleTask(DashboardRetrieveAndDoInfeasibleTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_incidents_per_agent: int = 4, + min_incidents_per_agent: int = 1, + num_agents: int = 4, + question: str = "", + dashboard_class: AbstractServiceNowTask = None, + function: callable = None, + provide_reason: bool = True, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.incident_hashtag = ( + f"#INC{str(id(self) % (10**8)).zfill(9)}" # identifier to select problems + ) + self.chart_title = f"Incidents with hashtag {self.incident_hashtag}" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + dashboard_config={ + "url": "/now/nav/ui/classic/params/target/sys_report", + "chart_title": self.chart_title, + "question": question, + "chart_series": "", + }, + level=level, + dashboard_class=dashboard_class, + ) + self.question = question + self.max_incidents_per_agent = max_incidents_per_agent + self.min_incidents_per_agent = min_incidents_per_agent + self.num_agents = num_agents + if (self.max_incidents_per_agent - self.min_incidents_per_agent) < 2 or self.num_agents < 2: + raise Exception( + "The difference between maximum incidents and minimum incidents should be at least two. The number of agents should also be at least 2." + ) + self.task_description = f"Retrieve the information mentioned in the following description from the report of the incidents with the title {self.incident_hashtag}. Using the information, follow the subsequent task steps mentioned. For all calculations, round of to the next highest integer first. For multiple modes, choose the highest value.\n" + if self.level == 3: + self.task_description += f"Follow the '{self.protocol_name}' protocol from the knowledge base for extra instructions.\n" + self.short_description = "Retrieve incident information and perform the mentioned task" + self.function = partial(function, provide_reason=provide_reason) + + def create_report( + self, + user_roles=["itil"], + ) -> None: + self.agents = {} + self.agent_sysids = [] + for _ in range(self.num_agents): + agent_response = create_user( + instance=self.instance, + first_name=f"{fake.first_name()}-{fake.first_name()}", + last_name=f"{fake.last_name()}-{fake.last_name()}", + return_full_response=True, + user_roles=user_roles, + ) + self.agents[agent_response["sys_id"]] = agent_response + self.agent_sysids.append(agent_response["sys_id"]) + + highest_agent = self.agent_sysids[ + -1 + ] # Choose last agent as the agent with maximum incidents + self.agents[highest_agent]["num_incidents"] = self.max_incidents_per_agent + self.agents[highest_agent]["incident_configs"] = [] + + lowest_agent = self.agent_sysids[ + 0 + ] # Choose first agent as the agent with minimum incidents + self.agents[lowest_agent]["num_incidents"] = self.min_incidents_per_agent + self.agents[lowest_agent]["incident_configs"] = [] + + for agent_sysid in self.agent_sysids[1:-1]: + self.agents[agent_sysid]["num_incidents"] = self.random.randint( + self.min_incidents_per_agent + 1, self.max_incidents_per_agent - 1 + ) + self.agents[agent_sysid]["incident_configs"] = [] + + number_assignments = sum([agent["num_incidents"] for agent in self.agents.values()]) + + all_existing_incidents = table_api_call( + instance=self.instance, table="incident", method="GET" + )["result"] + self.all_incident_numbers = [incident["number"] for incident in all_existing_incidents] + + self.new_incident_numbers = [] + for _ in range(number_assignments): + incident_number = ( + self.prefix + str(id(self) % (10**8)).zfill(8)[:4] + str(random.randint(1000, 9999)) + ) + while ( + incident_number in self.all_incident_numbers + or incident_number in self.new_incident_numbers + ): + incident_number = ( + self.prefix + + str(id(self) % (10**8)).zfill(8)[:4] + + str(random.randint(1000, 9999)) + ) + self.new_incident_numbers.append(incident_number) + + incident_number_idx = 0 + for agent, agent_attributes in self.agents.items(): + for _ in range(agent_attributes["num_incidents"]): + incident_response = create_incident( + instance=self.instance, + incident_number=self.new_incident_numbers[incident_number_idx], + caller_sys_id=self._base_user_sysid, + category="software", + priority=4, + impact=2, # priority is calculated as some combination of impact and urgency + urgency=3, + incident_hastag=self.incident_hashtag, + assigned_to=agent, + ) + self.agents[agent]["incident_configs"].append(incident_response) + incident_number_idx += 1 + + self.report_sys_id, _ = create_report( + instance=self.instance, + table="incident", + filter_hashtag=self.incident_hashtag, + field="assigned_to", + plot_title=self.chart_title, + random=self.random, + ) + + def get_agent_values(self, attribute_name, filter_than) -> list[str]: + agent_values = [] + agent_value_sysids = [] + agent_incidents = [ + agent_attributes["num_incidents"] for agent_attributes in self.agents.values() + ] + + if self.question == "max": + agent_value_sysids.append(self.agents[self.agent_sysids[-1]]["sys_id"]) + if attribute_name == "assigned_to": + agent_full_name = ( + self.agents[self.agent_sysids[-1]]["first_name"] + + " " + + self.agents[self.agent_sysids[-1]]["last_name"] + ) + agent_values.append(agent_full_name) + elif attribute_name == "first_name": + agent_first_name = self.agents[self.agent_sysids[-1]]["first_name"] + agent_values.append(agent_first_name) + else: + raise Exception("Filter column not supported.") + elif self.question == "min": + agent_value_sysids.append(self.agents[self.agent_sysids[0]]["sys_id"]) + if attribute_name == "assigned_to": + agent_full_name = ( + self.agents[self.agent_sysids[0]]["first_name"] + + " " + + self.agents[self.agent_sysids[0]]["last_name"] + ) + agent_values.append(agent_full_name) + elif attribute_name == "first_name": + agent_first_name = self.agents[self.agent_sysids[0]]["first_name"] + agent_values.append(agent_first_name) + else: + raise Exception("Filter column not supported.") + elif self.question == "mean" or self.question == "median" or self.question == "mode": + if self.question == "mean": + mean_incidents = np.mean(agent_incidents) + incidents_count = int(np.ceil(mean_incidents)) + elif self.question == "median": + incidents_count = int(np.ceil(np.median(agent_incidents))) + elif self.question == "mode": + # We select the maximum value if there are two or more modes + frequencies = {} + for count in agent_incidents: + if count not in frequencies: + frequencies[count] = 1 + else: + frequencies[count] += 1 + sorted_frequencies = { + count: frequency + for count, frequency in sorted( + frequencies.items(), key=lambda item: item[1], reverse=True + ) + } + max_frequency = list(sorted_frequencies.values())[0] + max_frequencies = [ + count + for count, frequency in sorted_frequencies.items() + if frequency == max_frequency + ] + incidents_count = int(max(max_frequencies)) + + for agent_sysid, agent_attributes in self.agents.items(): + if ( + filter_than == "greater" + and agent_attributes["num_incidents"] >= incidents_count + ) or ( + filter_than == "lesser" and agent_attributes["num_incidents"] <= incidents_count + ): + agent_value_sysids.append(agent_sysid) + if attribute_name == "assigned_to": + agent_full_name = ( + agent_attributes["first_name"] + " " + agent_attributes["last_name"] + ) + agent_values.append(agent_full_name) + + elif attribute_name == "first_name": + agent_first_name = agent_attributes["first_name"] + agent_values.append(agent_first_name) + else: + raise Exception("Filter column not supported.") + else: + raise Exception("Unsopprted question type.") + + return agent_values, agent_value_sysids + + def set_compositional_task(self) -> None: + raise NotImplementedError + + def teardown(self) -> None: + # Delete the report + db_delete_from_table( + instance=self.instance, + table="sys_report", + sys_id=self.report_sys_id, + ) + # Delete the incidents and users + for agent_sys_id in self.agents: + for incident in self.agents[agent_sys_id]["incident_configs"]: + db_delete_from_table( + instance=self.instance, + table="incident", + sys_id=incident["sys_id"], + ) + db_delete_from_table( + instance=self.instance, + table="sys_user", + sys_id=agent_sys_id, + ) + return super().teardown() + + +class DashboardRetrieveCatalogAndDoTask(DashboardRetrieveAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_items: int = 5, + min_items: int = 3, + question: str = "", + dashboard_class: AbstractServiceNowTask = None, + min_catalog_item: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.catalog_hashtag = ( + f"#CAT{str(id(self) % (10**8)).zfill(9)}" # identifier to select problems + ) + self.chart_title = f"Catalog with hashtag {self.catalog_hashtag}" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + dashboard_config={ + "url": "/now/nav/ui/classic/params/target/sys_report", + "chart_title": self.chart_title, + "question": question, + "chart_series": "", + }, + level=level, + dashboard_class=dashboard_class, + ) + self.question = question + self.max_number_per_item = self.random.choice([5, 6, 7]) + self.min_number_per_item = self.random.choice([1, 2]) + self.max_items = max_items + self.min_items = min_items + if self.max_items < 2 or self.min_items < 2: + raise Exception("The items allowed should at least be 2.") + self.min_catalog_item = min_catalog_item + self.task_description = f"You have to retrieve some information from a dashboard chart based on the description below. The chart presents the number of 'hardware items' available in stock. After retrieving the information, you will be asked to use it to complete a task.\n \n" + self.task_description += f"Title of the report: {self.catalog_hashtag}\n\n" + if self.level == 3: + self.task_description += f"Referring to the company protocol '{self.protocol_name}' (located in the 'Company Protocols' knowledge base), complete the dashboard retrieval task.\n\n" + self.short_description = ( + f"Retrieve information from the chart with the title {self.catalog_hashtag} and perform the mentioned task." + + "\nFor calculations, please round off to the next highest integer if required. If the required calculation has multiple possible answers (for example, 'mode' or 'most frequently' occuring value), please consider the highest value.\n\n" + ) + + def get_catalog_item_sysid(self, catalog_item: str) -> str: + catalog_item_response = table_api_call( + instance=self.instance, + table="sc_cat_item", + params={"sysparm_query": f"sys_name={catalog_item}", "sysparm_fields": "sys_id"}, + method="GET", + )["result"] + if len(catalog_item_response) == 0: + raise Exception("Catalog item not found.") + elif len(catalog_item_response) > 1: + raise Exception("Multiple catalog items found.") + return catalog_item_response[0]["sys_id"] + + def create_report( + self, + user_roles=["itil"], + ) -> None: + catalog_item_list = list(META_CONFIGS.keys()) + catalog_item_list.remove(self.min_catalog_item) + random_service_catalog_items = self.random.choice( + catalog_item_list, self.random.randint(self.min_items, self.max_items), replace=False + ).tolist() + cat_item_sys_name = { + "Developer Laptop (Mac)": "Developer Laptop (Mac)", + "iPad mini": "iPad mini", + "iPad pro": "iPad pro", + "Sales Laptop": "Sales Laptop", + "Standard Laptop": "Standard Laptop", + "Apple Watch": "Apple Watch", + "Apple MacBook Pro 15": 'Apple MacBook Pro 15"', + "Development Laptop (PC)": "Development Laptop (PC)", + "Loaner Laptop": "Notebook Computer Loaner", + } + + # shuffle + self.random.shuffle(random_service_catalog_items) + self.random_service_catalog_items = random_service_catalog_items + random_service_catalog_items = [self.min_catalog_item] + random_service_catalog_items + + service_catalog_report_config = {} + service_catalog_report_config[random_service_catalog_items[0]] = { + "quantity": self.min_number_per_item, + "description": META_CONFIGS[random_service_catalog_items[0]]["desc"], + "configuration": {}, + "item": random_service_catalog_items[0], + "sys_id": self.get_catalog_item_sysid( + cat_item_sys_name[random_service_catalog_items[0]] + ), + } + service_catalog_report_config[random_service_catalog_items[-1]] = { + "quantity": self.max_number_per_item, + "description": META_CONFIGS[random_service_catalog_items[-1]]["desc"], + "configuration": {}, + "item": random_service_catalog_items[-1], + "sys_id": self.get_catalog_item_sysid( + cat_item_sys_name[random_service_catalog_items[-1]] + ), + } + + for service_catalog_item in random_service_catalog_items[1:-1]: + service_catalog_report_config[service_catalog_item] = { + "quantity": self.random.randint( + self.min_number_per_item + 1, self.max_number_per_item - 1 + ), + "description": META_CONFIGS[service_catalog_item]["desc"], + "configuration": {}, + "item": service_catalog_item, + "sys_id": self.get_catalog_item_sysid(cat_item_sys_name[service_catalog_item]), + } + + self.service_catalog_report_config = service_catalog_report_config + created_request_items = [] + for ( + service_catalog_item, + service_catalog_item_config, + ) in service_catalog_report_config.items(): + for _ in range(service_catalog_item_config["quantity"]): + request_item_dict = { + "requested_for": self._base_user_sysid, + "quantity": 1, + "cat_item": service_catalog_item_config["sys_id"], + } + criteria_response = table_api_call( + instance=self.instance, + table="sc_req_item", + json=request_item_dict, + method="POST", + )["result"] + created_request_items.append((service_catalog_item, criteria_response["sys_id"])) + + self.created_request_items = created_request_items + + user_details = table_api_call( + instance=self.instance, + table="sys_user", + params={ + "sysparm_query": f"sys_id={self._base_user_sysid}", + "sysparm_fields": "first_name,last_name", + }, + method="GET", + )["result"][0] + user_full_name = user_details["first_name"] + " " + user_details["last_name"] + + self.report_sys_id, _ = create_report( + instance=self.instance, + table="sc_req_item", + filter_hashtag=user_full_name, + filter_field="requested_for", + field="cat_item", + plot_title=self.chart_title, + random=self.random, + ) + + def get_order_quantity_value(self) -> list[str]: + quantities = [ + service_catalog_report_config_attribute["quantity"] + for service_catalog_report_config_attribute in self.service_catalog_report_config.values() + ] + if self.question == "max": + if max(quantities) != self.max_number_per_item: + raise Exception("Maximum of quantities does not match attribute. Please check.") + target_quantity = self.max_number_per_item + elif self.question == "mean": + mean_quantity = np.mean(quantities) + target_quantity = int(np.ceil(mean_quantity)) + elif self.question == "median": + target_quantity = int(np.ceil(np.median(quantities))) + elif self.question == "mode": + frequencies = {} + for count in quantities: + if count not in frequencies: + frequencies[count] = 1 + else: + frequencies[count] += 1 + sorted_frequencies = { + count: frequency + for count, frequency in sorted( + frequencies.items(), key=lambda item: item[1], reverse=True + ) + } + max_frequency = list(sorted_frequencies.values())[0] + max_frequencies = [ + count + for count, frequency in sorted_frequencies.items() + if frequency == max_frequency + ] + target_quantity = int(max(max_frequencies)) + if target_quantity - self.min_number_per_item <= 0: + raise Exception("Unable to order quantity {target_quantity - self.min_number_per_item}") + return int(target_quantity - self.min_number_per_item) + + def set_compositional_task(self) -> None: + + order_config = { + "configuration": {}, + "description": META_CONFIGS[self.min_catalog_item]["desc"], + "item": self.min_catalog_item, + "quantity": self.get_order_quantity_value(), + } + + create_order_item_subtask = [ + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Service Catalog", + "url": "/now/nav/ui/classic/params/target/catalog_home.do", + }, + is_validated=False, + used_in_level_2=True, + ), + self.order_item_class( + instance=self.instance, + fixed_config=order_config, + is_validated=True, + used_in_level_2=True, + ), + ] + + self.compositional_task = create_order_item_subtask + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + + if self.level == 3: + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of all the items in stock.\n\n" + + f"\t - Task: Place an order for the least available item in stock. The quantity of the order should be such that the final quantity of this item matches the above retrieved value.\n" + + f"\t For example, consider the above task asks you to retrieve the maximum number of items in stock, say 4, and the least available item is an Apple Watch and its quantity is 1. You have to order 3 more Apple Watches.\n\n" + + f"\t - Please do not change any other configuration while placing the order for the item. You can find important links to the pages in the protocol article.\n\n" + + self.final_private_task_instructions + ) + + goal, info = super().setup_goal( + page=page, config=config, build_pretty_print_description=False + ) + + if self.level == 2: + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page.\n" + + f"\n2. Given the title of the report, search for it on this page.\n" + + f"\n3. Find the value which is the {self.description_mapping[self.question]} of the items present in stock as per the chart. Also remember the least available item in the stock.\n" + + f"\n4. Navigate to Self-Service > Service Catalog. \n" + + f"\n5. For the least available item in stock, place an order for extra items such that its quantity matches the value you found." + + "\nFor example, if you were requested to find the maximum value across the items, you would place an order for the least available item such that its NEW quantity matches this number. Please do not change any 'configuration' when placing the order.\n" + ) + + return goal, info + + def teardown(self) -> None: + # Delete the report + db_delete_from_table( + instance=self.instance, + table="sys_report", + sys_id=self.report_sys_id, + ) + # Delete the request items + for created_request_item in self.created_request_items: + db_delete_from_table( + instance=self.instance, + table="sc_req_item", + sys_id=created_request_item[1], + ) + return super().teardown() + + +class DashboardRetrieveCatalogAndDoInfeasibleTask(DashboardRetrieveAndDoInfeasibleTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_items: int = 5, + min_items: int = 3, + question: str = "", + dashboard_class: AbstractServiceNowTask = None, + min_catalog_item: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.catalog_hashtag = ( + f"#CAT{str(id(self) % (10**8)).zfill(9)}" # identifier to select problems + ) + self.chart_title = f"Catalog with hashtag {self.catalog_hashtag}" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + dashboard_config={ + "url": "/now/nav/ui/classic/params/target/sys_report", + "chart_title": self.chart_title, + "question": question, + "chart_series": "", + }, + level=level, + dashboard_class=dashboard_class, + ) + self.question = question + self.max_number_per_item = self.random.choice([5, 6, 7]) + self.min_number_per_item = self.random.choice([1, 2]) + self.max_items = max_items + self.min_items = min_items + if self.max_items < 2 or self.min_items < 2: + raise Exception("The items allowed should at least be 2.") + self.task_description = f"Retrieve the information mentioned in the following description from the report of the catalogs with the title {self.catalog_hashtag}. Using the information, follow the subsequent task steps mentioned. For all calculations, round of to the next highest integer first. For multiple modes, choose the highest value.\n" + if self.level == 3: + self.task_description += f"Follow the '{self.protocol_name}' protocol from the knowledge base for extra instructions.\n" + self.short_description = "Retrieve catalog information and perform the mentioned task" + self.min_catalog_item = min_catalog_item + self.function = partial( + get_infeasible_service_catalog_config, provide_reason=provide_reason + ) + self.all_configs = self.all_configs() + + @classmethod + def all_configs(cls) -> List[dict]: + with open(cls.config_path, "r") as f: + return json.load(f) + + def get_catalog_item_sysid(self, catalog_item: str) -> str: + catalog_item_response = table_api_call( + instance=self.instance, + table="sc_cat_item", + params={"sysparm_query": f"sys_name={catalog_item}", "sysparm_fields": "sys_id"}, + method="GET", + )["result"] + if len(catalog_item_response) == 0: + raise Exception("Catalog item not found.") + elif len(catalog_item_response) > 1: + raise Exception("Multiple catalog items found.") + return catalog_item_response[0]["sys_id"] + + def create_report( + self, + user_roles=["itil"], + ) -> None: + catalog_item_list = list(META_CONFIGS.keys()) + catalog_item_list.remove(self.min_catalog_item) + random_service_catalog_items = self.random.choice( + catalog_item_list, self.random.randint(self.min_items, self.max_items), replace=False + ).tolist() + cat_item_sys_name = { + "Developer Laptop (Mac)": "Developer Laptop (Mac)", + "iPad mini": "iPad mini", + "iPad pro": "iPad pro", + "Sales Laptop": "Sales Laptop", + "Standard Laptop": "Standard Laptop", + "Apple Watch": "Apple Watch", + "Apple MacBook Pro 15": 'Apple MacBook Pro 15"', + "Development Laptop (PC)": "Development Laptop (PC)", + "Loaner Laptop": "Notebook Computer Loaner", + } + + # shuffle + self.random.shuffle(random_service_catalog_items) + random_service_catalog_items = [ + self.min_catalog_item + ] + random_service_catalog_items.tolist() + self.random_service_catalog_items = random_service_catalog_items + + service_catalog_report_config = {} + service_catalog_report_config[random_service_catalog_items[0]] = { + "quantity": self.min_number_per_item, + "description": META_CONFIGS[random_service_catalog_items[0]]["desc"], + "configuration": {}, + "item": random_service_catalog_items[0], + "sys_id": self.get_catalog_item_sysid( + cat_item_sys_name[random_service_catalog_items[0]] + ), + } + service_catalog_report_config[random_service_catalog_items[-1]] = { + "quantity": self.max_number_per_item, + "description": META_CONFIGS[random_service_catalog_items[-1]]["desc"], + "configuration": {}, + "item": random_service_catalog_items[-1], + "sys_id": self.get_catalog_item_sysid( + cat_item_sys_name[random_service_catalog_items[-1]] + ), + } + + for service_catalog_item in random_service_catalog_items[1:-1]: + service_catalog_report_config[service_catalog_item] = { + "quantity": self.random.randint( + self.min_number_per_item + 1, self.max_number_per_item - 1 + ), + "description": META_CONFIGS[service_catalog_item]["desc"], + "configuration": {}, + "item": service_catalog_item, + "sys_id": self.get_catalog_item_sysid(cat_item_sys_name[service_catalog_item]), + } + + self.service_catalog_report_config = service_catalog_report_config + created_request_items = [] + for ( + service_catalog_item, + service_catalog_item_config, + ) in service_catalog_report_config.items(): + for _ in range(service_catalog_item_config["quantity"]): + request_item_dict = { + "requested_for": self._base_user_sysid, + "quantity": 1, + "cat_item": service_catalog_item_config["sys_id"], + } + criteria_response = table_api_call( + instance=self.instance, + table="sc_req_item", + json=request_item_dict, + method="POST", + )["result"] + created_request_items.append((service_catalog_item, criteria_response["sys_id"])) + + self.created_request_items = created_request_items + + user_details = table_api_call( + instance=self.instance, + table="sys_user", + params={ + "sysparm_query": f"sys_id={self._base_user_sysid}", + "sysparm_fields": "first_name,last_name", + }, + method="GET", + )["result"][0] + user_full_name = user_details["first_name"] + " " + user_details["last_name"] + + self.report_sys_id, _ = create_report( + instance=self.instance, + table="sc_req_item", + filter_hashtag=user_full_name, + filter_field="requested_for", + field="cat_item", + plot_title=self.chart_title, + random=self.random, + ) + + def get_order_quantity_value(self) -> list[str]: + quantities = [ + service_catalog_report_config_attribute["quantity"] + for service_catalog_report_config_attribute in self.service_catalog_report_config.values() + ] + if self.question == "max": + if max(quantities) != self.max_number_per_item: + raise Exception("Maximum of quantities does not match attribute. Please check.") + target_quantity = self.max_number_per_item + elif self.question == "mean": + mean_quantity = np.mean(quantities) + target_quantity = int(np.ceil(mean_quantity)) + elif self.question == "median": + target_quantity = int(np.ceil(np.median(quantities))) + elif self.question == "mode": + frequencies = {} + for count in quantities: + if count not in frequencies: + frequencies[count] = 1 + else: + frequencies[count] += 1 + sorted_frequencies = { + count: frequency + for count, frequency in sorted( + frequencies.items(), key=lambda item: item[1], reverse=True + ) + } + max_frequency = list(sorted_frequencies.values())[0] + max_frequencies = [ + count + for count, frequency in sorted_frequencies.items() + if frequency == max_frequency + ] + target_quantity = int(max(max_frequencies)) + if target_quantity - self.min_number_per_item <= 0: + raise Exception("Unable to order quantity {target_quantity - self.min_number_per_item}") + return int(target_quantity - self.min_number_per_item) + + def set_compositional_task(self) -> None: + + config = self.random.choice(self.all_configs) + self.configuration = config["configuration"] + order_config = { + "configuration": self.configuration, + "description": META_CONFIGS[self.min_catalog_item]["desc"], + "item": self.min_catalog_item, + "quantity": self.get_order_quantity_value(), + } + order_config, self.infeasible_reasons = self.function( + config=order_config, random=self.random + ) + + create_order_item_subtask = [ + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Service Catalog", + "url": "/now/nav/ui/classic/params/target/catalog_home.do", + }, + is_validated=False, + used_in_level_2=True, + has_description=False, + ), + self.order_item_class( + instance=self.instance, + fixed_config=order_config, + is_validated=False, + used_in_level_2=True, + ), + ] + + self.compositional_task = create_order_item_subtask + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + if self.level == 3: + self.task_description = ( + self.task_description + + f"Value to retrieve: {self.description_mapping[self.question]} of all the catalog items.\n" + + f"Task: Place an order for requesting more of the least available item in the report. The quantity of the order should be such that the final quantity of this item matches the above retrieved value.\n\n" + + self.final_private_task_instructions + ) + + goal, info = super().setup_goal( + page=page, config=config, build_pretty_print_description=False + ) + + if self.level == 2: + goal = ( + self.task_description + + f"\n1. Navigate to the CMDB reports and look for the catalog report with the mentioned hashtag. \n" + + f"\n2. Find the value which is the {self.description_mapping[self.question]} of the catalog items present in stock shown in the report. \n" + + f"\n3. Navigate to Self-Service > Service Catalog. \n" + + f"\n4. For the least available item in stock, place an order for extra items such that its quantity matches the value you found.\n" + ) + + return goal, info + + def teardown(self) -> None: + # Delete the report + db_delete_from_table( + instance=self.instance, + table="sys_report", + sys_id=self.report_sys_id, + ) + # Delete the request items + for created_request_item in self.created_request_items: + db_delete_from_table( + instance=self.instance, + table="sc_req_item", + sys_id=created_request_item[1], + ) + return super().teardown() + + +class DashDoFinalTask: + """Base class for dash do final tasks block tasks. Used to include these tasks across multiple superclasses.""" + + pass diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_catalog.py b/src/browsergym/workarena/tasks/compositional/dash_do_catalog.py new file mode 100644 index 0000000..859745c --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_catalog.py @@ -0,0 +1,1127 @@ +from .dash_do_base import DashboardRetrieveCatalogAndDoTask, DashDoFinalTask + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask, ReportMeanMedianModeRetrievalTask + +from ...instance import SNowInstance + +from browsergym.workarena.tasks.service_catalog import ( + OrderDeveloperLaptopTask, + OrderIpadMiniTask, + OrderIpadProTask, + OrderSalesLaptopTask, + OrderStandardLaptopTask, + OrderAppleWatchTask, + OrderAppleMacBookPro15Task, + OrderDevelopmentLaptopPCTask, + OrderLoanerLaptopTask, +) + + +class DashboardRetrieveCatalogAndOrderDeveloperLaptopTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderDeveloperLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Developer Laptop (Mac)", + ) + + +class DashboardRetrieveCatalogAndMaxOrderDeveloperLaptopTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderDeveloperLaptopTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 3, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderDeveloperLaptopTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderDeveloperLaptopTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderiPadMiniTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderIpadMiniTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="iPad mini", + ) + + +class DashboardRetrieveCatalogAndMaxOrderiPadMiniTask( + DashboardRetrieveCatalogAndOrderiPadMiniTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderiPadMiniTask( + DashboardRetrieveCatalogAndOrderiPadMiniTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderiPadMiniTask( + DashboardRetrieveCatalogAndOrderiPadMiniTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderiPadMiniTask( + DashboardRetrieveCatalogAndOrderiPadMiniTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderiPadProTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderIpadProTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="iPad pro", + ) + + +class DashboardRetrieveCatalogAndMaxOrderiPadProTask( + DashboardRetrieveCatalogAndOrderiPadProTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderiPadProTask( + DashboardRetrieveCatalogAndOrderiPadProTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderiPadProTask( + DashboardRetrieveCatalogAndOrderiPadProTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderiPadProTask( + DashboardRetrieveCatalogAndOrderiPadProTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderSalesLaptopTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderSalesLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Sales Laptop", + ) + + +class DashboardRetrieveCatalogAndMaxOrderSalesLaptopTask( + DashboardRetrieveCatalogAndOrderSalesLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderSalesLaptopTask( + DashboardRetrieveCatalogAndOrderSalesLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderSalesLaptopTask( + DashboardRetrieveCatalogAndOrderSalesLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderSalesLaptopTask( + DashboardRetrieveCatalogAndOrderSalesLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderStandardLaptopTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderStandardLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Standard Laptop", + ) + + +class DashboardRetrieveCatalogAndMaxOrderStandardLaptopTask( + DashboardRetrieveCatalogAndOrderStandardLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderStandardLaptopTask( + DashboardRetrieveCatalogAndOrderStandardLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderStandardLaptopTask( + DashboardRetrieveCatalogAndOrderStandardLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderStandardLaptopTask( + DashboardRetrieveCatalogAndOrderStandardLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderAppleWatchTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderAppleWatchTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Apple Watch", + ) + + +class DashboardRetrieveCatalogAndMaxOrderAppleWatchTask( + DashboardRetrieveCatalogAndOrderAppleWatchTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderAppleWatchTask( + DashboardRetrieveCatalogAndOrderAppleWatchTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderAppleWatchTask( + DashboardRetrieveCatalogAndOrderAppleWatchTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderAppleWatchTask( + DashboardRetrieveCatalogAndOrderAppleWatchTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = 0, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderAppleMacbookPro15Task(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderAppleMacBookPro15Task + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Apple MacBook Pro 15", + ) + + +class DashboardRetrieveCatalogAndMaxOrderAppleMacbookPro15Task( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15Task, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderAppleMacbookPro15Task( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15Task, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderAppleMacbookPro15Task( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15Task, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderAppleMacbookPro15Task( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15Task, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderDevelopmentLaptopPCTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Development Laptop (PC)", + ) + + +class DashboardRetrieveCatalogAndMaxOrderDevelopmentLaptopPCTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderDevelopmentLaptopPCTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderDevelopmentLaptopPCTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderDevelopmentLaptopPCTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +class DashboardRetrieveCatalogAndOrderLoanerLaptopTask(DashboardRetrieveCatalogAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderLoanerLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Loaner Laptop", + ) + + +class DashboardRetrieveCatalogAndMaxOrderLoanerLaptopTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + ) + + +class DashboardRetrieveCatalogAndMeanOrderLoanerLaptopTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + ) + + +class DashboardRetrieveCatalogAndMedianOrderLoanerLaptopTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + ) + + +class DashboardRetrieveCatalogAndModeOrderLoanerLaptopTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] + +DASH_AND_ORDER = [ + DashboardRetrieveCatalogAndMaxOrderDeveloperLaptopTask, + DashboardRetrieveCatalogAndMaxOrderiPadMiniTask, + DashboardRetrieveCatalogAndMaxOrderiPadProTask, + DashboardRetrieveCatalogAndMaxOrderSalesLaptopTask, + DashboardRetrieveCatalogAndMaxOrderStandardLaptopTask, + DashboardRetrieveCatalogAndMaxOrderAppleWatchTask, + DashboardRetrieveCatalogAndMaxOrderAppleMacbookPro15Task, + DashboardRetrieveCatalogAndMaxOrderDevelopmentLaptopPCTask, + DashboardRetrieveCatalogAndMaxOrderLoanerLaptopTask, +] +DASH_COMPUTE_MEAN_AND_ORDER = [ + DashboardRetrieveCatalogAndMeanOrderDeveloperLaptopTask, + DashboardRetrieveCatalogAndMeanOrderiPadMiniTask, + DashboardRetrieveCatalogAndMeanOrderiPadProTask, + DashboardRetrieveCatalogAndMeanOrderSalesLaptopTask, + DashboardRetrieveCatalogAndMeanOrderStandardLaptopTask, + DashboardRetrieveCatalogAndMeanOrderAppleWatchTask, + DashboardRetrieveCatalogAndMeanOrderAppleMacbookPro15Task, + DashboardRetrieveCatalogAndMeanOrderDevelopmentLaptopPCTask, + DashboardRetrieveCatalogAndMeanOrderLoanerLaptopTask, +] + +DASH_COMPUTE_MEDIAN_AND_ORDER = [ + DashboardRetrieveCatalogAndMedianOrderDeveloperLaptopTask, + DashboardRetrieveCatalogAndMedianOrderiPadMiniTask, + DashboardRetrieveCatalogAndMedianOrderiPadProTask, + DashboardRetrieveCatalogAndMedianOrderSalesLaptopTask, + DashboardRetrieveCatalogAndMedianOrderStandardLaptopTask, + DashboardRetrieveCatalogAndMedianOrderAppleWatchTask, + DashboardRetrieveCatalogAndMedianOrderAppleMacbookPro15Task, + DashboardRetrieveCatalogAndMedianOrderDevelopmentLaptopPCTask, + DashboardRetrieveCatalogAndMedianOrderLoanerLaptopTask, +] + +DASH_COMPUTE_MODE_AND_ORDER = [ + DashboardRetrieveCatalogAndModeOrderDeveloperLaptopTask, + DashboardRetrieveCatalogAndModeOrderiPadMiniTask, + DashboardRetrieveCatalogAndModeOrderiPadProTask, + DashboardRetrieveCatalogAndModeOrderSalesLaptopTask, + DashboardRetrieveCatalogAndModeOrderStandardLaptopTask, + DashboardRetrieveCatalogAndModeOrderAppleWatchTask, + DashboardRetrieveCatalogAndModeOrderAppleMacbookPro15Task, + DashboardRetrieveCatalogAndModeOrderDevelopmentLaptopPCTask, + DashboardRetrieveCatalogAndModeOrderLoanerLaptopTask, +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_catalog_infeasible.py b/src/browsergym/workarena/tasks/compositional/dash_do_catalog_infeasible.py new file mode 100644 index 0000000..9e9069e --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_catalog_infeasible.py @@ -0,0 +1,2047 @@ +from .dash_do_base import DashboardRetrieveCatalogAndDoInfeasibleTask, DashDoFinalTask + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask, ReportMeanMedianModeRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...config import ( + ORDER_DEVELOPER_LAPTOP_TASK_CONFIG_PATH, + ORDER_IPAD_MINI_TASK_CONFIG_PATH, + ORDER_IPAD_PRO_TASK_CONFIG_PATH, + ORDER_SALES_LAPTOP_TASK_CONFIG_PATH, + ORDER_STANDARD_LAPTOP_TASK_CONFIG_PATH, + ORDER_APPLE_WATCH_TASK_CONFIG_PATH, + ORDER_APPLE_MAC_BOOK_PRO15_TASK_CONFIG_PATH, + ORDER_DEVELOPMENT_LAPTOP_PC_TASK_CONFIG_PATH, + ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH, +) +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.service_catalog import ( + OrderDeveloperLaptopTask, + OrderIpadMiniTask, + OrderIpadProTask, + OrderSalesLaptopTask, + OrderStandardLaptopTask, + OrderAppleWatchTask, + OrderAppleMacBookPro15Task, + OrderDevelopmentLaptopPCTask, + OrderLoanerLaptopTask, +) + + +class DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_DEVELOPER_LAPTOP_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderDeveloperLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Developer Laptop (Mac)", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderDeveloperLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderDeveloperLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderDeveloperLaptopWithReasonInfeasibleTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderDeveloperLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderDeveloperLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderDeveloperLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderDeveloperLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderDeveloperLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderDeveloperLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_IPAD_MINI_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderIpadMiniTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="iPad mini", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderiPadMiniInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderiPadMiniInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderiPadMiniInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderiPadMiniInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderiPadMiniInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderiPadMiniInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderiPadMiniInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderiPadMiniInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadMiniInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_IPAD_PRO_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderIpadProTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="iPad pro", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderiPadProInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderiPadProInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderiPadProInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderiPadProInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderiPadProInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderiPadProInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderiPadProInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderiPadProInfeasibleTask( + DashboardRetrieveCatalogAndOrderiPadProInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_SALES_LAPTOP_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderSalesLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Sales Laptop", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderSalesLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderSalesLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderSalesLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderSalesLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderSalesLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderSalesLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderSalesLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderSalesLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderSalesLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_STANDARD_LAPTOP_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderStandardLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Standard Laptop", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderStandardLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderStandardLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderStandardLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderStandardLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderStandardLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderStandardLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderStandardLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderStandardLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderStandardLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_APPLE_WATCH_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderAppleWatchTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Apple Watch", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderAppleWatchInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderAppleWatchInfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderAppleWatchInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderAppleWatchInfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderAppleWatchInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderAppleWatchInfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderAppleWatchInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderAppleWatchInfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleWatchInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_APPLE_MAC_BOOK_PRO15_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderAppleMacBookPro15Task + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Apple MacBook Pro 15", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderAppleMacbookPro15InfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderAppleMacbookPro15InfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderAppleMacbookPro15InfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderAppleMacbookPro15InfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderAppleMacbookPro15InfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderAppleMacbookPro15InfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderAppleMacbookPro15InfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderAppleMacbookPro15InfeasibleTask( + DashboardRetrieveCatalogAndOrderAppleMacbookPro15InfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_DEVELOPMENT_LAPTOP_PC_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderDevelopmentLaptopPCTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Development Laptop (PC)", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderDevelopmentLaptopPCInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderDevelopmentLaptopPCInfeasibleTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderDevelopmentLaptopPCInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderDevelopmentLaptopPCInfeasibleTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderDevelopmentLaptopPCInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderDevelopmentLaptopPCInfeasibleTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderDevelopmentLaptopPCInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderDevelopmentLaptopPCInfeasibleTask( + DashboardRetrieveCatalogAndOrderDevelopmentLaptopPCInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask( + DashboardRetrieveCatalogAndDoInfeasibleTask +): + config_path = ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH + + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + dashboard_class: AbstractServiceNowTask = ReportMinMaxRetrievalTask, + question: str = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + self.order_item_class = OrderLoanerLaptopTask + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=dashboard_class, + question=question, + min_catalog_item="Loaner Laptop", + provide_reason=provide_reason, + ) + + +class DashboardRetrieveCatalogAndMaxOrderLoanerLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMaxOrderLoanerLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMinMaxRetrievalTask, + question="max", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMeanOrderLoanerLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMeanOrderLoanerLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mean", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndMedianOrderLoanerLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndMedianOrderLoanerLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="median", + provide_reason=False, + ) + + +class DashboardRetrieveCatalogAndModeOrderLoanerLaptopInfeasibleWithReasonTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=True, + ) + + +class DashboardRetrieveCatalogAndModeOrderLoanerLaptopInfeasibleTask( + DashboardRetrieveCatalogAndOrderLoanerLaptopInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve information based on incidents from the dashboard and do the task. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + dashboard_class=ReportMeanMedianModeRetrievalTask, + question="mode", + provide_reason=False, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_create_incident.py b/src/browsergym/workarena/tasks/compositional/dash_do_create_incident.py new file mode 100644 index 0000000..be04874 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_create_incident.py @@ -0,0 +1,403 @@ +import random +from playwright.sync_api._generated import Page +from typing import Tuple + +from .dash_do_base import DashboardRetrieveIncidentAndDoTask, DashDoFinalTask + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask, ReportMeanMedianModeRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.form import CreateIncidentTask + + +class DashboardRetrieveIncidentAndCreateIncidentTask(DashboardRetrieveIncidentAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + self.filter_than = "lesser" + self.attribute_name = "assigned_to" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + self.prefix = "DCI" + + def set_compositional_task(self) -> None: + # The unique name for the user is created once the task is instantiated + base_user = table_api_call( + instance=self.instance, + table="sys_user", + params={ + "sysparm_query": f"sys_id={self._base_user_sysid}", + }, + )["result"][0] + self.user_name = base_user["first_name"] + " " + base_user["last_name"] + + agent_full_names, agent_value_sysids = self.get_agent_values( + self.attribute_name, self.filter_than + ) + self.agent_value_sysids = agent_value_sysids + incident_numbers = [] + for _ in range(len(agent_full_names)): + incident_number = "INC" + str(random.randint(1000000, 9999999)) + while ( + incident_number in self.all_incident_numbers or incident_number in incident_numbers + ): + incident_number = "INC" + str(random.randint(1000000, 9999999)) + incident_numbers.append(incident_number) + + self.incident_numbers = incident_numbers + + create_incident_subtasks = [] + + for agent_full_name, incident_number in zip(agent_full_names, incident_numbers): + self.incident_short_description = "Compulsory training for employee in probation" + incident_config = { + "fields": { + "caller_id": "Caller", + "category": "Category", + "short_description": "Short description", + "impact": "Impact", + "number": "Number", + "urgency": "Urgency", + "assigned_to": "Assigned to", + }, + "task_fields": [ + "caller_id", + "category", + "short_description", + "impact", + "number", + "urgency", + "assigned_to", + ], + "template_record": { + "caller_id": self.user_name, + "category": "Inquiry / Help", + "short_description": self.incident_short_description, + "impact": "1 - High", + "number": incident_number, + "urgency": "1 - High", + "assigned_to": agent_full_name, + }, + } + + create_incident_subtask = [ + # Navigate to the incident list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Service Desk", + "module": "Incidents", + "url": "/now/nav/ui/classic/params/target/incident_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create an incident + CreateIncidentTask( + instance=self.instance, + fixed_config=incident_config, + is_validated=False, + used_in_level_2=True, + check_record_created=False, + ), + ] + create_incident_subtasks += create_incident_subtask + + self.compositional_task = create_incident_subtasks + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + incident_numbers_string = ", ".join(self.incident_numbers) + + if self.level == 3: + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of the number of incidents assigned across agents. Using this value, retrieve agents that have less than or equal number of incidents assigned to them compared to this value.\n" + + f"\t For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have less than or equal to 2 assigned incidents.\n\n" + + f"\t - Task: For each agent that fits the above criteria, create an 'incident' with the following information (only fill these fields and the 'incident number' from below) and assign it to them using the 'Assigned to' field: \n" + + f"\t\t Category: 'Inquiry / Help', Impact: '1 - High', Urgency: '1 - High', Short description: 'Compulsory training for employee in probation', Caller: '{self.user_name}'. Make sure to use an 'incident number' from the list as described below.\n" + + f"\t Importantly, you should override the default incident numbers in the form and instead use one incident number from the following list for each incident you create: {incident_numbers_string}.\n\n" + + f"Note that you will create as many incidents as there are agents matching the above criteria.\n" + + f"\t For example, consider the above case and say you have 3 agents with less than or equal to 2 incidents assigned to them in the chart. You will be creating '3' new incidents here, one assigned to each agent. \n\n" + ) + + goal, info = super().setup_goal(page=page, config=config) + + if self.level == 2: + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page. \n" + + f"\n2. Given the title of the report, search for it on this page. The report shows the number of 'incidents' assigned to an 'agent'.\n" + + f"\n3. Find the agents with a number of incidents assigned less than or equal to the {self.description_mapping[self.question]} of the number of assigned incidents across agents. For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have less than or equal to 2 assigned incidents.\n" + + f"\n4. Navigate to Service Desk > Incidents. \n" + + f"\n5. You have to create new 'incidents' from this page for all the agents based on the above criteria. Only fill the following fields when creating a new incident:- Category: 'Inquiry / Help', Impact: '1 - High', Urgency: '1 - High', Short description: 'Compulsory training for employee in probation', Caller: '{self.user_name}' and 'assign' them to each agent using the 'Assigned to' field.\n\n" + + f"Importantly, you should override the default incident numbers in the form and instead use one incident number from the following list for each incident you create: {incident_numbers_string}.\n" + + f"Note that you will create as many incidents as there are agents matching the above criteria." + + "\nFor example, consider the above case and say you have 3 agents with less than or equal to 2 incidents assigned to them in the chart. You will be creating '3' new incidents here, one assigned to each agent. \n" + ) + + return goal, info + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + fixed_template_record = { + "short_description": "Compulsory training for employee in probation", + "impact": "1", + "urgency": "1", + } + for incident_number in self.incident_numbers: + created_incident_response = table_api_call( + instance=self.instance, + table="incident", + params={ + "sysparm_query": f"number={incident_number}", + "sysparm_fields": "assigned_to,impact,urgency,short_description,description", + }, + method="GET", + )["result"] + if len(created_incident_response) == 0: + return ( + 0, + False, + "", + {"message": f"No incident created with number {incident_number}."}, + ) + elif len(created_incident_response) > 1: + return ( + 0, + False, + "", + {"message": f"Multiple incidents created with number {incident_number}."}, + ) + created_incident_response = created_incident_response[0] + if created_incident_response["assigned_to"]["value"] not in self.agent_value_sysids: + return ( + 0, + False, + "", + {"message": f"Incident {incident_number} assigned to a random agent."}, + ) + + for key, value in fixed_template_record.items(): + if str(created_incident_response[key]).lower() != str(value).lower(): + return ( + 0, + False, + "", + { + "message": f"Incident {incident_number} assigned incorrect value to field {key}." + }, + ) + + for agent_sysid in self.agent_value_sysids: + created_incident_response = table_api_call( + instance=self.instance, + table="incident", + params={ + "sysparm_query": f"assigned_to={agent_sysid}^short_description={self.incident_short_description}", + "sysparm_fields": "assigned_to", + }, + method="GET", + )["result"] + if len(created_incident_response) == 0: + return ( + 0, + False, + "", + { + "message": f"No incident assigned to agent {self.agents[agent_sysid]['user_name']}." + }, + ) + elif len(created_incident_response) > 1: + return ( + 0, + False, + "", + { + "message": f"Multiple incidents assigned to agent {self.agents[agent_sysid]['user_name']}." + }, + ) + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + + def teardown(self) -> None: + for incident_number in self.incident_numbers: + created_incident_response = table_api_call( + instance=self.instance, + table="incident", + params={ + "sysparm_query": f"number={incident_number}", + }, + method="GET", + )["result"] + if len(created_incident_response) > 1: + raise Exception("Multiple incidents created") + if len(created_incident_response) == 1: + created_incident_sysid = created_incident_response[0]["sys_id"] + db_delete_from_table( + instance=self.instance, + table="incident", + sys_id=created_incident_sysid, + ) + + return super().teardown() + + +class DashboardRetrieveIncidentAndMinCreateIncidentTask( + DashboardRetrieveIncidentAndCreateIncidentTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanCreateIncidentTask( + DashboardRetrieveIncidentAndCreateIncidentTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianCreateIncidentTask( + DashboardRetrieveIncidentAndCreateIncidentTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeCreateIncidentTask( + DashboardRetrieveIncidentAndCreateIncidentTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] + +DASH_AND_CREATE_INCIDENT = [ + DashboardRetrieveIncidentAndMinCreateIncidentTask, +] +DASH_COMPUTE_AND_CREATE_INCIDENT = [ + DashboardRetrieveIncidentAndMeanCreateIncidentTask, + DashboardRetrieveIncidentAndMedianCreateIncidentTask, + DashboardRetrieveIncidentAndModeCreateIncidentTask, +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_create_incident_infeasible.py b/src/browsergym/workarena/tasks/compositional/dash_do_create_incident_infeasible.py new file mode 100644 index 0000000..854a1fd --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_create_incident_infeasible.py @@ -0,0 +1,278 @@ +import random +from playwright.sync_api._generated import Page +from typing import Tuple + +from .dash_do_base import DashboardRetrieveIncidentAndDoInfeasibleTask, DashDoFinalTask +from .utils.infeasible_configs import get_infeasible_form_config + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.form import CreateIncidentTask + + +class DashboardRetrieveIncidentAndCreateIncidentInfeasibleTask( + DashboardRetrieveIncidentAndDoInfeasibleTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + provide_reason: bool = True, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + function=get_infeasible_form_config, + provide_reason=provide_reason, + ) + self.task_description = "" + self.short_description = ( + "Create an incident for the worst performing employee from the list." + ) + self.filter_than = "lesser" + self.attribute_name = "assigned_to" + self.prefix = "ICI" + + def set_compositional_task(self) -> None: + # The unique name for the user is created once the task is instantiated + base_user = table_api_call( + instance=self.instance, + table="sys_user", + params={ + "sysparm_query": f"sys_id={self._base_user_sysid}", + }, + )["result"][0] + self.user_name = base_user["first_name"] + " " + base_user["last_name"] + + agent_full_names, agent_value_sysids = self.get_agent_values( + self.attribute_name, self.filter_than + ) + self.agent_value_sysids = agent_value_sysids + incident_numbers = [] + for _ in range(len(agent_full_names)): + incident_number = "INC" + str(random.randint(1000000, 9999999)) + while ( + incident_number in self.all_incident_numbers or incident_number in incident_numbers + ): + incident_number = "INC" + str(random.randint(1000000, 9999999)) + incident_numbers.append(incident_number) + + self.incident_numbers = incident_numbers + + create_incident_subtasks = [] + + for agent_full_name, incident_number in zip(agent_full_names, incident_numbers): + incident_config = { + "fields": { + "caller_id": "Caller", + "category": "Category", + "description": "Description", + "short_description": "Short description", + "impact": "Impact", + "number": "Number", + "urgency": "Urgency", + "assigned_to": "Assigned to", + }, + "task_fields": [ + "caller_id", + "category", + "description", + "short_description", + "impact", + "number", + "urgency", + "assigned_to", + ], + "template_record": { + "caller_id": self.user_name, + "category": "Inquiry / Help", + "description": "Compulsory training for employee in probation", + "short_description": "Compulsory training for employee in probation", + "impact": "1 - High", + "number": incident_number, + "urgency": "1 - High", + "assigned_to": agent_full_name, + }, + "infeasible_task_fields": [ + "category", + "description", + "short_description", + "impact", + "number", + ], + } + + incident_config, self.infeasible_reasons = self.function( + config=incident_config, random=self.random + ) + + create_incident_subtask = [ + # Navigate to the incident list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Service Desk", + "module": "Incidents", + "url": "/now/nav/ui/classic/params/target/incident_list.do", + }, + is_validated=False, + used_in_level_2=True, + has_description=False, + ), + # Create an incident + CreateIncidentTask( + instance=self.instance, + fixed_config=incident_config, + is_validated=False, + used_in_level_2=True, + check_record_created=False, + has_description=True, + ), + ] + create_incident_subtasks += create_incident_subtask + + self.compositional_task = create_incident_subtasks + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + incident_numbers_string = ", ".join(self.incident_numbers) + + if self.level == 3: + self.task_description = ( + self.task_description + + f"Value to retrieve: {self.description_mapping[self.question]} of all the incidents. Comparator: Less than or equal to the value.\n" + + f"Task: Create incidents with the following information: \n" + + f"Category: 'Inquiry / Help', Impact: '1 - High', Urgency: '1 - High', Short description: 'Compulsory training for employee in probation', Caller: '{self.user_name}'.\n" + + f"Assign the incidents you create to the agents mentioned above. You can use the incident numbers: {incident_numbers_string}, one for each." + ) + + goal, info = super().setup_goal( + page=page, config=config, build_pretty_print_description=False + ) + + if self.level == 2: + goal = ( + self.task_description + + f"\n1. Navigate to the CMDB reports and look for the report with the mentioned hashtag. \n" + + f"\n2. Find the agents with number of incidents less than or equal to the {self.description_mapping[self.question]} of the incidents assigned to every one. \n" + + f"\n3. Navigate to Service Desk > Incidents. \n" + + f"\n4. Create new incidents with the following field values:- Category: 'Inquiry / Help', Impact: '1 - High', Urgency: '1 - High', Short description: 'Compulsory training for employee in probation' and assign them to each of the agents. You will create as many incidents as there are agents.\n" + + f"\nYou should use the following incident numbers for each incident (one for each): {incident_numbers_string}." + ) + + return goal, info + + def teardown(self) -> None: + for incident_number in self.incident_numbers: + created_incident_response = table_api_call( + instance=self.instance, + table="incident", + params={ + "sysparm_query": f"number={incident_number}", + }, + method="GET", + )["result"] + if len(created_incident_response) > 1: + raise Exception("Multiple incidents created") + if len(created_incident_response) == 1: + created_incident_sysid = created_incident_response[0]["sys_id"] + db_delete_from_table( + instance=self.instance, + table="incident", + sys_id=created_incident_sysid, + ) + + return super().teardown() + + +class DashboardRetrieveIncidentAndMinCreateIncidentInfeasibleWithReasonTask( + DashboardRetrieveIncidentAndCreateIncidentInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 3, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMinCreateIncidentInfeasibleTask( + DashboardRetrieveIncidentAndCreateIncidentInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create an incident to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create an incident for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_create_problem.py b/src/browsergym/workarena/tasks/compositional/dash_do_create_problem.py new file mode 100644 index 0000000..b2dd935 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_create_problem.py @@ -0,0 +1,336 @@ +from playwright.sync_api._generated import Page +from typing import Tuple + +from .dash_do_base import DashboardRetrieveIncidentAndDoTask, DashDoFinalTask + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask, ReportMeanMedianModeRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.form import CreateProblemTask + + +class DashboardRetrieveIncidentAndCreateProblemTask(DashboardRetrieveIncidentAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + self.attribute_name = "assigned_to" + self.filter_than = "lesser" + self.prefix = "DCP" + + def set_compositional_task(self) -> None: + agent_full_names, agent_value_sysids = self.get_agent_values( + self.attribute_name, self.filter_than + ) + self.agent_value_sysids = agent_value_sysids + create_problem_subtasks = [] + self.short_description = "Compulsory training for employee in probation" + for agent_full_name in agent_full_names: + problem_config = { + "fields": { + "short_description": "Problem statement", + "urgency": "Urgency", + "assigned_to": "Assigned to", + "impact": "Impact", + }, + "task_fields": [ + "short_description", + "urgency", + "assigned_to", + "impact", + ], + "template_record": { + "short_description": self.short_description, + "impact": "1 - High", + "urgency": "1 - High", + "assigned_to": agent_full_name, + }, + } + + create_problem_subtask = [ + # Navigate to the incident list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Problem", + "module": "All", + "url": "/now/nav/ui/classic/params/target/problem_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create a problem + CreateProblemTask( + instance=self.instance, + fixed_config=problem_config, + is_validated=False, + used_in_level_2=True, + check_record_created=False, + ), + ] + create_problem_subtasks += create_problem_subtask + + self.compositional_task = create_problem_subtasks + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report( + user_roles=[ + "itil", + "problem_admin", + "problem_manager", + "problem_coordinator", + "problem_task_analyst", + ] + ) + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + + if self.level == 3: + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of the number of incidents assigned across agents. Retrieve agents that have less than or equal number of incidents assigned to them compared to this value.\n" + + f"\t For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have less than or equal to 2 assigned incidents.\n\n" + + f"\t - Task: For each agent that fits the above criteria, create a 'problem' with the following information (only fill these fields) and assign it to them using the 'Assigned to' field: \n" + + f"\t\t Impact: '1 - High', Urgency: '1 - High', Problem statement: 'Compulsory training for employee in probation'.\n" + + f"Note that you will create as many problems as there are agents matching the above criteria.\n" + + f"\t For example, consider the above case and say you have 3 agents with less than or equal to 2 incidents assigned to them in the chart. You will be creating '3' problems here, one assigned to each agent. \n\n" + ) + + goal, info = super().setup_goal(page=page, config=config) + + if self.level == 2: + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page. \n" + + f"\n2. Given the title of the report, search for it on this page. The report shows the number of 'incidents' assigned to an 'agent'.\n" + + f"\n3. Find the agents with a number of incidents assigned less than or equal to the {self.description_mapping[self.question]} of the number of assigned incidents across agents. For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have less than or equal to 2 assigned incidents.\n" + + f"\n4. Navigate to All > Problems. \n" + + f"\n5. You have to create new 'problems' from this page for all the agents based on the above criteria. Only fill the following fields when creating a new problem:- Impact: '1 - High', Urgency: '1 - High', Problem statement: 'Compulsory training for employee in probation' and 'assign' them to each agent.\n\n" + + f"Note that you will create as many problems as there are agents matching the above criteria." + + "\nFor example, consider the above case and say you have 3 agents with less than or equal to 2 incidents assigned to them in the chart. You will be creating '3' problems here, one assigned to each agent. \n" + ) + + return goal, info + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + fixed_template_record = { + "short_description": "Compulsory training for employee in probation", + "impact": "1", + "urgency": "1", + } + for agent_sysid in self.agent_value_sysids: + created_problem_response = table_api_call( + instance=self.instance, + table="problem", + params={ + "sysparm_query": f"assigned_to={agent_sysid}^short_description={self.short_description}", + }, + method="GET", + )["result"] + if len(created_problem_response) == 0: + return ( + 0, + False, + "", + { + "message": f"No problem created for agent {self.agents[agent_sysid]['user_name']}." + }, + ) + elif len(created_problem_response) > 1: + return ( + 0, + False, + "", + { + "message": f"Multiple problems created for agent {self.agents[agent_sysid]['user_name']}." + }, + ) + created_problem_response = created_problem_response[0] + for key, value in fixed_template_record.items(): + if str(created_problem_response[key]).lower() != str(value).lower(): + return ( + 0, + False, + "", + { + "message": f"Problem for agent {self.agents[agent_sysid]['user_name']} assigned incorrect value to field {key}." + }, + ) + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + + def teardown(self) -> None: + for agent_sysid in self.agent_sysids: + created_problem_response = table_api_call( + instance=self.instance, + table="problem", + params={ + "sysparm_query": f"assigned_to={agent_sysid}", + }, + method="GET", + )["result"] + for problem in created_problem_response: + db_delete_from_table( + instance=self.instance, + table="problem", + sys_id=problem["sys_id"], + ) + return super().teardown() + + +class DashboardRetrieveIncidentAndMinCreateProblemTask( + DashboardRetrieveIncidentAndCreateProblemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanCreateProblemTask( + DashboardRetrieveIncidentAndCreateProblemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianCreateProblemTask( + DashboardRetrieveIncidentAndCreateProblemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeCreateProblemTask( + DashboardRetrieveIncidentAndCreateProblemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] + +DASH_AND_CREATE_PROBLEM = [DashboardRetrieveIncidentAndMinCreateProblemTask] +DASH_COMPUTE_AND_CREATE_PROBLEM = [ + DashboardRetrieveIncidentAndMeanCreateProblemTask, + DashboardRetrieveIncidentAndMedianCreateProblemTask, + DashboardRetrieveIncidentAndModeCreateProblemTask, +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_create_problem_infeasible.py b/src/browsergym/workarena/tasks/compositional/dash_do_create_problem_infeasible.py new file mode 100644 index 0000000..044ba3c --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_create_problem_infeasible.py @@ -0,0 +1,235 @@ +from playwright.sync_api._generated import Page +from typing import Tuple + +from .dash_do_base import DashboardRetrieveIncidentAndDoInfeasibleTask, DashDoFinalTask +from .utils.infeasible_configs import get_infeasible_form_config + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.form import CreateProblemTask + + +class DashboardRetrieveIncidentAndCreateProblemInfeasibleTask( + DashboardRetrieveIncidentAndDoInfeasibleTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + function=get_infeasible_form_config, + provide_reason=provide_reason, + ) + self.task_description = "" + self.short_description = "Create a problem for the worst performing employee from the list." + self.attribute_name = "assigned_to" + self.filter_than = "lesser" + self.prefix = "ICP" + + def set_compositional_task(self) -> None: + agent_full_names, agent_value_sysids = self.get_agent_values( + self.attribute_name, self.filter_than + ) + self.agent_value_sysids = agent_value_sysids + create_problem_subtasks = [] + for agent_full_name in agent_full_names: + problem_config = { + "fields": { + "short_description": "Problem statement", + "description": "Description", + "urgency": "Urgency", + "assigned_to": "Assigned to", + "impact": "Impact", + }, + "task_fields": [ + "short_description", + "description", + "urgency", + "assigned_to", + "impact", + ], + "template_record": { + "description": "Compulsory training for employee in probation", + "short_description": "Compulsory training for employee in probation", + "impact": "1 - High", + "urgency": "1 - High", + "assigned_to": agent_full_name, + }, + "infeasible_task_fields": ["short_description", "description", "urgency", "impact"], + } + problem_config, self.infeasible_reasons = self.function( + config=problem_config, random=self.random + ) + + create_problem_subtask = [ + # Navigate to the incident list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Problem", + "module": "All", + "url": "/now/nav/ui/classic/params/target/problem_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create a problem + CreateProblemTask( + instance=self.instance, + fixed_config=problem_config, + is_validated=False, + used_in_level_2=True, + check_record_created=False, + ), + ] + create_problem_subtasks += create_problem_subtask + + self.compositional_task = create_problem_subtasks + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report( + user_roles=[ + "itil", + "problem_admin", + "problem_manager", + "problem_coordinator", + "problem_task_analyst", + ] + ) + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + if self.level == 3: + self.task_description = ( + self.task_description + + f"Value to retrieve: {self.description_mapping[self.question]} of all the incidents. Comparator: Less than or equal to the value.\n" + + f"Task: Create problems with the following information: \n" + + f"Category: 'Inquiry / Help', Impact: '1 - High', Urgency: '1 - High', Short description: 'Compulsory training for employee in probation'.\n" + + f"Assign the problems you create to the agents mentioned above." + ) + + goal, info = super().setup_goal(page=page, config=config) + + if self.level == 2: + goal = ( + self.task_description + + f"\n1. Navigate to the CMDB reports and look for the report with the mentioned hashtag. \n" + + f"\n2. Find the agents with number of incidents less than or equal to the {self.description_mapping[self.question]} of the incidents assigned to every one. \n" + + f"\n3. Navigate to All > Problems. \n" + + f"\n4. Create new problems with the following field values:- Category: 'Inquiry / Help', Impact: '1 - High', Urgency: '1 - High', Short description: 'Compulsory training for employee in probation' and assign them to each of the agents." + + "\nYou will create as many problems as there are agents.\n" + ) + + return goal, info + + def teardown(self) -> None: + for agent_sysid in self.agent_sysids: + created_problem_response = table_api_call( + instance=self.instance, + table="problem", + params={ + "sysparm_query": f"assigned_to={agent_sysid}", + }, + method="GET", + )["result"] + for problem in created_problem_response: + db_delete_from_table( + instance=self.instance, + table="problem", + sys_id=problem["sys_id"], + ) + return super().teardown() + + +class DashboardRetrieveIncidentAndMinCreateProblemInfeasibleWithReasonTask( + DashboardRetrieveIncidentAndCreateProblemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMinCreateProblemInfeasibleTask( + DashboardRetrieveIncidentAndCreateProblemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the worst performing employee and create a problem to assign them a probation course. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a problem for the worst performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_filter.py b/src/browsergym/workarena/tasks/compositional/dash_do_filter.py new file mode 100644 index 0000000..7d46636 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_filter.py @@ -0,0 +1,1600 @@ +import random +from playwright.sync_api._generated import Page + +from .dash_do_base import DashboardRetrieveIncidentAndDoTask, DashDoFinalTask + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask, ReportMeanMedianModeRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.list import ( + FilterAssetListTask, + FilterHardwareListTask, + FilterIncidentListTask, + FilterUserListTask, +) + + +class DashboardRetrieveIncidentAndFilterListTask(DashboardRetrieveIncidentAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a list based on their assignments" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + self.prefix = "DIF" + + def get_filter_config(self, attribute_name, filter_than) -> dict: + filter_values, agent_value_sysids = self.get_agent_values( + attribute_name=attribute_name, filter_than=filter_than + ) + self.agent_value_sysids = agent_value_sysids + if len(filter_values) == 1: + filter_kind = "AND" + else: + filter_kind = "OR" + + filter_config = { + "filter_columns": [self.attribute_name] * len(filter_values), + "filter_kind": filter_kind, + "filter_values": filter_values, + } + return filter_config + + +class DashboardRetrieveIncidentAndFilterAssetListTask(DashboardRetrieveIncidentAndFilterListTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_assets_per_agent: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + self.max_assets_per_agent = max_assets_per_agent + self.attribute_name = "assigned_to" + + def set_compositional_task(self) -> None: + filter_config = self.get_filter_config( + attribute_name=self.attribute_name, filter_than=self.filter_than + ) + + filter_asset_list_subtask = [ + # Navigate to the asset list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Asset", + "module": "Portfolios > All Assets", + "url": "/now/nav/ui/classic/params/target/alm_asset_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter asset list + FilterAssetListTask( + is_validated=True, + list_name="alm_asset", + used_in_level_2=True, + fixed_config=filter_config, + ), + ] + + self.compositional_task = filter_asset_list_subtask + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + + # We create dummy assets for the consumable and license categories here + ### NOTE: We can create assets without any of the following information, save time, and still assign them to the user. The task should be fine. + consumable_category_sysid = table_api_call( + instance=self.instance, + table="cmdb_model_category", + method="GET", + params={"sysparm_query": "asset_class=alm_consumable", "sysparm_fields": "sys_id"}, + )["result"][0]["sys_id"] + + consumables = table_api_call( + instance=self.instance, + table="cmdb_model", + method="GET", + params={ + "sysparm_query": f"cmdb_model_category={consumable_category_sysid}", + "sysparm_fields": "sys_id", + }, + )["result"] + consumables_sysids = [consumable["sys_id"] for consumable in consumables] + + license_category_sysid = table_api_call( + instance=self.instance, + table="cmdb_model_category", + method="GET", + params={"sysparm_query": "asset_class=alm_license", "sysparm_fields": "sys_id"}, + )["result"][0]["sys_id"] + + licenses = table_api_call( + instance=self.instance, + table="cmdb_model", + method="GET", + params={ + "sysparm_query": f"cmdb_model_category={license_category_sysid}", + "sysparm_fields": "sys_id", + }, + )["result"][:10] + license_sysids = [license["sys_id"] for license in licenses] + + self.new_asset_sys_ids = [] + for agent_sysid in self.agent_sysids: + num_assets = self.random.choice(range(1, self.max_assets_per_agent)) + for _ in range(num_assets): + consumable_asset_data = { + "asset_tag": "CONSUMABLE" + str(random.randint(100, 999)), + "model": self.random.choice(consumables_sysids), + "model_category": consumable_category_sysid, + "assigned_to": agent_sysid, + "cost": 1000.00, + "purchase_date": "2024-05-08", + "substatus": "in_use", + } + response = table_api_call( + instance=self.instance, + table="alm_asset", + json=consumable_asset_data, + method="POST", + ) + self.new_asset_sys_ids.append(response["result"]["sys_id"]) + license_asset_data = { + "asset_tag": "LICENSE" + str(random.randint(100, 999)), + "model": self.random.choice(license_sysids), + "model_category": license_category_sysid, + "assigned_to": agent_sysid, + "cost": 1000.00, + "purchase_date": "2024-05-08", + "substatus": "in_use", + } + response = table_api_call( + instance=self.instance, + table="alm_asset", + json=license_asset_data, + method="POST", + ) + self.new_asset_sys_ids.append(response["result"]["sys_id"]) + + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + + if self.level == 3: + filter_than = f"{self.filter_than} than or " if self.filter_than else "" + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of the number of incidents assigned across agents. Retrieve agents that have {filter_than}equal number of incidents assigned to them compared to this value.\n" + + f"\t For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have {filter_than}equal to 2 assigned incidents.\n\n" + + f"\t - Task: Filter the Asset List using the {self.attribute_name} field corresponding to the agents that fit the criteria above. \n" + + f"The list is present at Portfolios > All Assets. \n\n" + + self.final_private_task_instructions + ) + + goal, info = super().setup_goal( + page=page, config=config, build_pretty_print_description=False + ) + + if self.level == 2: + if self.filter_than: + step_3 = f"\n3. Find the agents with number of incidents {self.filter_than} than or equal to the {self.description_mapping[self.question]} value of the number of incidents assigned across agents. \n" + else: + step_3 = f"\n3. Find the agent with the {self.description_mapping[self.question]} assigned incidents. \n" + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page. \n" + + f"\n2. Given the title of the report, search for it on this page. The report shows the number of 'incidents' assigned to an 'agent'.\n" + + step_3 + + f"\n4. Navigate to Portfolios > All Assets. \n" + + f"\nUsing the field {self.attribute_name} for the agent/ agents that fit the critera above, filter the list.\n" + ) + + return goal, info + + def teardown(self) -> None: + # Delete all assets + for new_asset_sysid in self.new_asset_sys_ids: + db_delete_from_table( + instance=self.instance, + table="alm_asset", + sys_id=new_asset_sysid, + ) + return super().teardown() + + +class DashboardRetrieveIncidentAndFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterListTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_assets_per_agent: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.max_assets_per_agent = max_assets_per_agent + self.attribute_name = "assigned_to" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + + def set_compositional_task(self) -> None: + + filter_config = self.get_filter_config( + attribute_name=self.attribute_name, filter_than=self.filter_than + ) + + filter_hardware_asset_list_subtask = [ + # Navigate to the hardware asset list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + "url": "/now/nav/ui/classic/params/target/alm_hardware_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter hardware list + FilterHardwareListTask( + is_validated=True, + list_name="alm_hardware", + used_in_level_2=True, + fixed_config=filter_config, + ), + ] + + self.compositional_task = filter_hardware_asset_list_subtask + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + + hardware_category_sysid = table_api_call( + instance=self.instance, + table="cmdb_model_category", + method="GET", + params={"sysparm_query": "asset_class=alm_hardware", "sysparm_fields": "sys_id"}, + )["result"][0]["sys_id"] + + hardwares = table_api_call( + instance=self.instance, + table="cmdb_model", + method="GET", + params={ + "sysparm_query": f"cmdb_model_category={hardware_category_sysid}", + "sysparm_fields": "sys_id", + }, + )["result"] + hardware_sysids = [hardware["sys_id"] for hardware in hardwares] + self.new_asset_sysids = [] + for agent_sysid in self.agent_sysids: + num_assets = self.random.choice(range(1, self.max_assets_per_agent)) + for _ in range(num_assets): + hardware_asset_data = { + "asset_tag": "CONSUMABLE" + str(random.randint(100, 999)), + "model": self.random.choice(hardware_sysids), + "model_category": hardware_category_sysid, + "assigned_to": agent_sysid, + "cost": 1000.00, + "purchase_date": "2024-05-08", + "substatus": "in_use", + } + response = table_api_call( + instance=self.instance, + table="alm_hardware", + json=hardware_asset_data, + method="POST", + ) + self.new_asset_sysids.append(response["result"]["sys_id"]) + + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + + if self.level == 3: + filter_than = f"{self.filter_than} than or " if self.filter_than else "" + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of the number of incidents assigned across agents. Retrieve agents that have {filter_than}equal number of incidents assigned to them compared to this value.\n" + + f"\t For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have {filter_than}equal to 2 assigned incidents.\n\n" + + f"\t - Task: Filter the Hardware List using the {self.attribute_name} field corresponding to the agents that fit the criteria above. \n" + + f"The list is present at Portfolios > Hardware Assets. \n\n" + + self.final_private_task_instructions + ) + + goal, info = super().setup_goal( + page=page, config=config, build_pretty_print_description=False + ) + + if self.level == 2: + if self.filter_than: + step_3 = f"\n3. Find the agents with number of incidents {self.filter_than} than or equal to the {self.description_mapping[self.question]} value of the number of incidents assigned across agents. \n" + else: + step_3 = f"\n3. Find the agent with the {self.description_mapping[self.question]} assigned incidents. \n" + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page. \n" + + f"\n2. Given the title of the report, search for it on this page. The report shows the number of 'incidents' assigned to an 'agent'.\n" + + step_3 + + f"\n4. Navigate to Portfolios > Hardware Assets. \n" + + f"\nUsing the field {self.attribute_name} for the agent/ agents that fit the critera above, filter the list.\n" + ) + + return goal, info + + def teardown(self) -> None: + # Delete all assets + for new_asset_sysid in self.new_asset_sysids: + db_delete_from_table( + instance=self.instance, + table="alm_hardware", + sys_id=new_asset_sysid, + ) + return super().teardown() + + +class DashboardRetrieveIncidentAndFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterListTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.attribute_name = "assigned_to" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + + def set_compositional_task(self) -> None: + + filter_config = self.get_filter_config( + attribute_name=self.attribute_name, filter_than=self.filter_than + ) + + filter_incident_list_subtask = [ + # Navigate to the incident list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Service Desk", + "module": "Incidents", + "url": "/now/nav/ui/classic/params/target/incident_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter incident list + FilterIncidentListTask( + is_validated=True, + list_name="incident", + used_in_level_2=True, + fixed_config=filter_config, + ), + ] + + self.compositional_task = filter_incident_list_subtask + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + + if self.level == 3: + filter_than = f"{self.filter_than} than or " if self.filter_than else "" + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of the number of incidents assigned across agents. Retrieve agents that have {filter_than}equal number of incidents assigned to them compared to this value.\n" + + f"\t For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have {filter_than}equal to 2 assigned incidents.\n\n" + + f"\t - Task: Filter the Incident List using the {self.attribute_name} field corresponding to the agents that fit the criteria above. \n" + + f"The list is present at Service Desk > Incidents. \n\n" + + self.final_private_task_instructions + ) + + goal, info = super().setup_goal( + page=page, config=config, build_pretty_print_description=False + ) + + if self.level == 2: + if self.filter_than: + step_3 = f"\n3. Find the agents with number of incidents {self.filter_than} than or equal to the {self.description_mapping[self.question]} value of the number of incidents assigned across agents. \n" + else: + step_3 = f"\n3. Find the agent with the {self.description_mapping[self.question]} assigned incidents. \n" + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page. \n" + + f"\n2. Given the title of the report, search for it on this page. The report shows the number of 'incidents' assigned to an 'agent'.\n" + + step_3 + + f"\n4. Navigate to Service Desk > Incidents. \n" + + f"\nUsing the field {self.attribute_name} for the agent/ agents that fit the critera above, filter the list.\n" + ) + + return goal, info + + +class DashboardRetrieveIncidentAndFilterUserListTask(DashboardRetrieveIncidentAndFilterListTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.attribute_name = "first_name" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + + def set_compositional_task(self) -> None: + filter_config = self.get_filter_config( + attribute_name=self.attribute_name, filter_than=self.filter_than + ) + + filter_user_list_subtask = [ + # Navigate to the user list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Organization", + "module": "Users", + "url": "/now/nav/ui/classic/params/target/sys_user_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter user list + FilterUserListTask( + is_validated=True, + list_name="user", + used_in_level_2=True, + fixed_config=filter_config, + ), + ] + + self.compositional_task = filter_user_list_subtask + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + + if self.level == 3: + filter_than = f"{self.filter_than} than or " if self.filter_than else "" + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of the number of incidents assigned across agents. Retrieve agents that have {filter_than}equal number of incidents assigned to them compared to this value.\n" + + f"\t For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have {filter_than}equal to 2 assigned incidents.\n\n" + + f"\t - Task: Filter the User List using the {self.attribute_name} field corresponding to the agents that fit the criteria above. \n" + + f"The list is present at Organization > Users. \n\n" + + self.final_private_task_instructions + ) + + goal, info = super().setup_goal( + page=page, config=config, build_pretty_print_description=False + ) + + if self.level == 2: + if self.filter_than: + step_3 = f"3. Find the agents with number of incidents {self.filter_than} than or equal to the {self.description_mapping[self.question]} value of the number of incidents assigned across agents. \n" + else: + step_3 = f"3. Find the agent with the {self.description_mapping[self.question]} assigned incidents. \n" + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page. \n" + + f"\n2. Given the title of the report, search for it on this page. The report shows the number of 'incidents' assigned to an 'agent'.\n" + + step_3 + + f"\n4. Navigate to Organization > Users. \n" + + f"\nUsing the field {self.attribute_name} for the agent/ agents that fit the critera above, filter the list.\n" + ) + + return goal, info + + +class DashboardRetrieveIncidentAndMaxFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMinFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanGreaterFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianGreaterFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeGreaterFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanLesserFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianLesserFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeLesserFilterAssetListTask( + DashboardRetrieveIncidentAndFilterAssetListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an asset list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list based on incidents assigned to an employee" + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMinFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanGreaterFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianGreaterFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeGreaterFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanLesserFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianLesserFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeLesserFilterHardwareListTask( + DashboardRetrieveIncidentAndFilterHardwareListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a hardware list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a hardware list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMinFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanGreaterFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianGreaterFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeGreaterFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanLesserFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianLesserFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeLesserFilterIncidentListTask( + DashboardRetrieveIncidentAndFilterIncidentListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter an incident list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter an incident list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMinFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = None + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="min", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanGreaterFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianGreaterFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeGreaterFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = "greater" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanLesserFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianLesserFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeLesserFilterUserListTask( + DashboardRetrieveIncidentAndFilterUserListTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best or worst performing agent and filter a user list based on their assignments. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Retrieve the best or worst performing agent and filter a user list based on their assignments." + """ + self.filter_than = "lesser" + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] + +DASH_COMPUTE_MAX_FILTER_LIST = [ + DashboardRetrieveIncidentAndMaxFilterAssetListTask, + DashboardRetrieveIncidentAndMaxFilterHardwareListTask, + DashboardRetrieveIncidentAndMaxFilterIncidentListTask, + DashboardRetrieveIncidentAndMaxFilterUserListTask, +] +DASH_COMPUTE_MIN_FILTER_LIST = [ + DashboardRetrieveIncidentAndMinFilterAssetListTask, + DashboardRetrieveIncidentAndMinFilterHardwareListTask, + DashboardRetrieveIncidentAndMinFilterIncidentListTask, + DashboardRetrieveIncidentAndMinFilterUserListTask, +] +DASH_COMPUTE_MEAN_FILTER_LIST = [ + DashboardRetrieveIncidentAndMeanGreaterFilterAssetListTask, + DashboardRetrieveIncidentAndMeanGreaterFilterHardwareListTask, + DashboardRetrieveIncidentAndMeanGreaterFilterIncidentListTask, + DashboardRetrieveIncidentAndMeanGreaterFilterUserListTask, + DashboardRetrieveIncidentAndMeanLesserFilterAssetListTask, + DashboardRetrieveIncidentAndMeanLesserFilterHardwareListTask, + DashboardRetrieveIncidentAndMeanLesserFilterIncidentListTask, + DashboardRetrieveIncidentAndMeanLesserFilterUserListTask, +] + +DASH_COMPUTE_MEDIAN_FILTER_LIST = [ + DashboardRetrieveIncidentAndMedianGreaterFilterAssetListTask, + DashboardRetrieveIncidentAndMedianLesserFilterAssetListTask, + DashboardRetrieveIncidentAndMedianGreaterFilterHardwareListTask, + DashboardRetrieveIncidentAndMedianLesserFilterHardwareListTask, + DashboardRetrieveIncidentAndMedianGreaterFilterIncidentListTask, + DashboardRetrieveIncidentAndMedianLesserFilterIncidentListTask, + DashboardRetrieveIncidentAndMedianGreaterFilterUserListTask, + DashboardRetrieveIncidentAndMedianLesserFilterUserListTask, +] + +DASH_COMPUTE_MODE_FILTER_LIST = [ + DashboardRetrieveIncidentAndModeGreaterFilterAssetListTask, + DashboardRetrieveIncidentAndModeLesserFilterAssetListTask, + DashboardRetrieveIncidentAndModeGreaterFilterHardwareListTask, + DashboardRetrieveIncidentAndModeLesserFilterHardwareListTask, + DashboardRetrieveIncidentAndModeGreaterFilterIncidentListTask, + DashboardRetrieveIncidentAndModeLesserFilterIncidentListTask, + DashboardRetrieveIncidentAndModeGreaterFilterUserListTask, + DashboardRetrieveIncidentAndModeLesserFilterUserListTask, +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_request_item.py b/src/browsergym/workarena/tasks/compositional/dash_do_request_item.py new file mode 100644 index 0000000..77baa57 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_request_item.py @@ -0,0 +1,1315 @@ +import random +from playwright.sync_api._generated import Page +from typing import Tuple + +from .dash_do_base import DashboardRetrieveIncidentAndDoTask, DashDoFinalTask + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask, ReportMeanMedianModeRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.form import CreateItemRequestTask + + +class DashboardRetrieveIncidentAndRequestItemTask(DashboardRetrieveIncidentAndDoTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + item: str = None, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + ) -> None: + """ + Retrieve the best performing agent and request an item for them. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. ""Order an item for the best performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + ) + if not item: + raise Exception("No item passed to assign") + self.item = item + self.attribute_name = "assigned_to" # Return full name + self.filter_than = "greater" + self.prefix = "DRI" + + def set_compositional_task(self) -> None: + # The unique name for the user is created once the task is instantiated + requested_items = table_api_call( + instance=self.instance, + table="sc_req_item", + method="GET", + )["result"] + current_requested_items_numbers = [ + requested_item["number"] for requested_item in requested_items + ] + + agent_full_names, agent_value_sysids = self.get_agent_values( + self.attribute_name, self.filter_than + ) + self.agent_value_sysids = agent_value_sysids + + requested_item_numbers = [] + + for _ in range(len(agent_full_names)): + requested_item_number = "RITM" + str(random.randint(1000000, 9999999)) + while ( + requested_item_number in current_requested_items_numbers + or requested_item_number in requested_item_numbers + ): + requested_item_number = "RITM" + str(random.randint(1000000, 9999999)) + requested_item_numbers.append(requested_item_number) + + self.requested_item_numbers = requested_item_numbers + + create_item_request_subtasks = [] + + for agent_full_name, requested_item_number in zip(agent_full_names, requested_item_numbers): + request_item_config = { + "fields": { + "number": "Number", + "cat_item": "Item", + "requested_for": "Requested for", + "quantity": "Quantity", + }, + "task_fields": ["number", "cat_item", "requested_for", "quantity"], + "template_record": { + "number": requested_item_number, + "cat_item": self.item, + "requested_for": agent_full_name, + "quantity": "1", + }, + } + + create_item_request_subtask = [ + # Navigate to the item request list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Open Records", + "module": "Open Records > Items", + "url": "/now/nav/ui/classic/params/target/sc_req_item_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create an item request + CreateItemRequestTask( + instance=self.instance, + fixed_config=request_item_config, + is_validated=False, + used_in_level_2=True, + check_record_created=False, + ), + ] + create_item_request_subtasks += create_item_request_subtask + + self.compositional_task = create_item_request_subtasks + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + requested_item_numbers_string = ", ".join(self.requested_item_numbers) + + if self.level == 3: + self.task_description = ( + self.task_description + + f"\t - Please retrieve the '{self.description_mapping[self.question]}' value of the number of incidents assigned across agents. Retrieve agents that have greater than or equal number of incidents assigned to them compared to this value.\n" + + f"\t For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have greater than or equal to 2 assigned incidents.\n\n" + + f"\t - Task: For each agent that fits the above criteria, create an 'item request' with the following information (only fill these fields and the 'request item number' from below) and 'request it' for them using the 'Requested for' field: \n" + + f"\t\t Item: {self.item}, Quantity: 1, Requested for: .\n" + + f"\t Importantly, you should override the default request item numbers in the form and instead use one request item number from the following list for each item request you create: {requested_item_numbers_string}.\n\n" + + f"Note that you will create as many item requests as there are agents matching the above criteria.\n" + + f"\t For example, consider the above case and say you have 3 agents with greater than or equal to 2 incidents assigned to them in the chart. You will be creating '3' item requests here, one for each agent. \n\n" + ) + + goal, info = super().setup_goal(page=page, config=config) + + if self.level == 2: + goal = ( + self.short_description + + f"\n1. Navigate to the Reports > View/Run page. \n" + + f"\n2. Given the title of the report, search for it on this page. The report shows the number of 'incidents' assigned to an 'agent'.\n" + + f"\n3. Find the agents with a number of incidents assigned greater than or equal to the {self.description_mapping[self.question]} of the number of assigned incidents across agents. For example, if you were asked to find the 'mean' or 'average' for a case where there are 4 agents assigned 1,2,3,2 incidents respectively, the mean would be 2. The list of agents required for solving the following task would be all the agents that have greater than or equal to 2 assigned incidents.\n" + + f"\n4. Navigate to Open Records > Items. \n" + + f"\n5. You have to create new 'item requests' from this page for all the agents based on the above criteria. Only fill the following fields when creating a new item request:- Item: {self.item}, Quantity: 1 and 'request' them for each agent using the 'Requested For' field.\n\n" + + f"Importantly, you should override the default request item numbers in the form and instead use one request item number from the following list for each item request you create: {requested_item_numbers_string}.\n" + + f"Note that you will create as many item requests as there are agents matching the above criteria.\n" + + "For example, consider the above case and say you have 3 agents with greater than or equal to 2 incidents assigned to them in the chart. You will be creating '3' item requests here, one for each agent. \n" + ) + + return goal, info + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + for requested_item_number in self.requested_item_numbers: + created_request_item_response = table_api_call( + instance=self.instance, + table="sc_req_item", + params={ + "sysparm_query": f"number={requested_item_number}", + "sysparm_fields": "requested_for,cat_item,quantity", + }, + method="GET", + )["result"] + if len(created_request_item_response) == 0: + return ( + 0, + False, + "", + {"message": f"No request item created with number {requested_item_number}."}, + ) + elif len(created_request_item_response) > 1: + return ( + 0, + False, + "", + { + "message": f"Multiple request items created with number {requested_item_number}." + }, + ) + created_request_item_response = created_request_item_response[0] + if ( + created_request_item_response["requested_for"]["value"] + not in self.agent_value_sysids + ): + return ( + 0, + False, + "", + { + "message": f"Request item {requested_item_number} created for a random agent." + }, + ) + if str(created_request_item_response["quantity"]) != "1": + return ( + 0, + False, + "", + { + "message": f"Request item {requested_item_number} requested incorrect number of items." + }, + ) + cat_item = created_request_item_response["cat_item"] + if not cat_item: + return ( + 0, + False, + "", + {"message": f"Request item {requested_item_number} did not request an item."}, + ) + cat_item_response = table_api_call( + instance=self.instance, + table="sc_cat_item", + params={ + "sysparm_query": f"sys_id={cat_item['value']}", + "sysparm_fields": "sys_name", + }, + method="GET", + )["result"] + if len(cat_item_response) == 0: + return ( + 0, + False, + "", + {"message": f"Request item {requested_item_number} did not request an item."}, + ) + + if cat_item_response[0]["sys_name"] != self.item: + return ( + 0, + False, + "", + { + "message": f"Request item {requested_item_number} requested an incorrect item." + }, + ) + + for agent_sysid in self.agent_value_sysids: + created_request_item_response = table_api_call( + instance=self.instance, + table="sc_req_item", + params={ + "sysparm_query": f"requested_for={agent_sysid}", + "sysparm_fields": "requested_for", + }, + method="GET", + )["result"] + if len(created_request_item_response) == 0: + return ( + 0, + False, + "", + { + "message": f"No request created for agent {self.agents[agent_sysid]['user_name']}." + }, + ) + elif len(created_request_item_response) > 1: + return ( + 0, + False, + "", + { + "message": f"Multiple requests created for agent {self.agents[agent_sysid]['user_name']}." + }, + ) + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + + def teardown(self) -> None: + for requested_item_number in self.requested_item_numbers: + created_item_request_response = table_api_call( + instance=self.instance, + table="sc_req_item", + params={ + "sysparm_query": f"number={requested_item_number}", + }, + method="GET", + )["result"] + if len(created_item_request_response) > 1: + raise Exception("Multiple request items created") + if len(created_item_request_response) == 1: + created_item_request_sysid = created_item_request_response[0]["sys_id"] + db_delete_from_table( + instance=self.instance, + table="sc_req_item", + sys_id=created_item_request_sysid, + ) + return super().teardown() + + +class DashboardRetrieveIncidentAndMaxRequestAppleWatchTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestAppleWatchTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestAppleWatchTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestAppleWatchTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleWatch2Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple Watch Series 2 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch Series 2", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestAppleWatch2Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple Watch Series 2 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch Series 2", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestAppleWatch2Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple Watch Series 2 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch Series 2", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestAppleWatch2Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple Watch Series 2 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch Series 2", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIpad3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPad 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPad 3", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestAppleIpad3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPad 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPad 3", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestAppleIpad3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPad 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPad 3", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestAppleIpad3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPad 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPad 3", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIphone13proTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 pro for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13 pro", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestAppleIphone13proTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 pro for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13 pro", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestAppleIphone13proTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 pro for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13 pro", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestAppleIphone13proTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 pro for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13 pro", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIphone13Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestAppleIphone13Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestAppleIphone13Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestAppleIphone13Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Apple iPhone 13 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestGalaxyNote20Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Galaxy Note 20 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Galaxy Note 20", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestGalaxyNote20Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Galaxy Note 20 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Galaxy Note 20", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestGalaxyNote20Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Galaxy Note 20 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Galaxy Note 20", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestGalaxyNote20Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Galaxy Note 20 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Galaxy Note 20", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestGoogleNexus7Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Google Nexus 7 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Google Nexus 7", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestGoogleNexus7Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Google Nexus 7 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Google Nexus 7", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestGoogleNexus7Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Google Nexus 7 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Google Nexus 7", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestGoogleNexus7Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Google Nexus 7 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Google Nexus 7", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestMicrosoftSurfacePro3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Microsoft Surface Pro 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Microsoft Surface Pro 3", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestMicrosoftSurfacePro3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Microsoft Surface Pro 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Microsoft Surface Pro 3", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestMicrosoftSurfacePro3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Microsoft Surface Pro 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Microsoft Surface Pro 3", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestMicrosoftSurfacePro3Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Microsoft Surface Pro 3 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Microsoft Surface Pro 3", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestPixel4aTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Pixel 4a for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Pixel 4a", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestPixel4aTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Pixel 4a for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Pixel 4a", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestPixel4aTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Pixel 4a for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Pixel 4a", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestPixel4aTask( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an Pixel 4a for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Pixel 4a", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMaxRequestWindowsSurfacePro4Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Windows Surface Pro 4 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Windows Surface Pro 4", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMeanRequestWindowsSurfacePro4Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Windows Surface Pro 4 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Windows Surface Pro 4", + question="mean", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndMedianRequestWindowsSurfacePro4Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Windows Surface Pro 4 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Windows Surface Pro 4", + question="median", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +class DashboardRetrieveIncidentAndModeRequestWindowsSurfacePro4Task( + DashboardRetrieveIncidentAndRequestItemTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request a Windows Surface Pro 4 for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Windows Surface Pro 4", + question="mode", + dashboard_class=ReportMeanMedianModeRetrievalTask, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] + + +DASH_AND_REQUEST = [ + DashboardRetrieveIncidentAndMaxRequestAppleWatchTask, + DashboardRetrieveIncidentAndMaxRequestAppleWatch2Task, + DashboardRetrieveIncidentAndMaxRequestAppleIpad3Task, + DashboardRetrieveIncidentAndMaxRequestAppleIphone13proTask, + DashboardRetrieveIncidentAndMaxRequestAppleIphone13Task, + DashboardRetrieveIncidentAndMaxRequestGalaxyNote20Task, + DashboardRetrieveIncidentAndMaxRequestGoogleNexus7Task, + DashboardRetrieveIncidentAndMaxRequestMicrosoftSurfacePro3Task, + DashboardRetrieveIncidentAndMaxRequestPixel4aTask, + DashboardRetrieveIncidentAndMaxRequestWindowsSurfacePro4Task, +] +DASH_COMPUTE_MEAN_AND_REQUEST = [ + DashboardRetrieveIncidentAndMeanRequestAppleWatchTask, + DashboardRetrieveIncidentAndMeanRequestAppleWatch2Task, + DashboardRetrieveIncidentAndMeanRequestAppleIpad3Task, + DashboardRetrieveIncidentAndMeanRequestAppleIphone13proTask, + DashboardRetrieveIncidentAndMeanRequestAppleIphone13Task, + DashboardRetrieveIncidentAndMeanRequestGalaxyNote20Task, + DashboardRetrieveIncidentAndMeanRequestGoogleNexus7Task, + DashboardRetrieveIncidentAndMeanRequestMicrosoftSurfacePro3Task, + DashboardRetrieveIncidentAndMeanRequestPixel4aTask, + DashboardRetrieveIncidentAndMeanRequestWindowsSurfacePro4Task, +] + +DASH_COMPUTE_MEDIAN_AND_REQUEST = [ + DashboardRetrieveIncidentAndMedianRequestAppleWatchTask, + DashboardRetrieveIncidentAndMedianRequestAppleWatch2Task, + DashboardRetrieveIncidentAndMedianRequestAppleIpad3Task, + DashboardRetrieveIncidentAndMedianRequestAppleIphone13proTask, + DashboardRetrieveIncidentAndMedianRequestAppleIphone13Task, + DashboardRetrieveIncidentAndMedianRequestGalaxyNote20Task, + DashboardRetrieveIncidentAndMedianRequestGoogleNexus7Task, + DashboardRetrieveIncidentAndMedianRequestMicrosoftSurfacePro3Task, + DashboardRetrieveIncidentAndMedianRequestPixel4aTask, + DashboardRetrieveIncidentAndMedianRequestWindowsSurfacePro4Task, +] + +DASH_COMPUTE_MODE_AND_REQUEST = [ + DashboardRetrieveIncidentAndModeRequestAppleWatchTask, + DashboardRetrieveIncidentAndModeRequestAppleWatch2Task, + DashboardRetrieveIncidentAndModeRequestAppleIpad3Task, + DashboardRetrieveIncidentAndModeRequestAppleIphone13proTask, + DashboardRetrieveIncidentAndModeRequestAppleIphone13Task, + DashboardRetrieveIncidentAndModeRequestGalaxyNote20Task, + DashboardRetrieveIncidentAndModeRequestGoogleNexus7Task, + DashboardRetrieveIncidentAndModeRequestMicrosoftSurfacePro3Task, + DashboardRetrieveIncidentAndModeRequestPixel4aTask, + DashboardRetrieveIncidentAndModeRequestWindowsSurfacePro4Task, +] diff --git a/src/browsergym/workarena/tasks/compositional/dash_do_request_item_infeasible.py b/src/browsergym/workarena/tasks/compositional/dash_do_request_item_infeasible.py new file mode 100644 index 0000000..6d8dae4 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/dash_do_request_item_infeasible.py @@ -0,0 +1,693 @@ +import random +from playwright.sync_api._generated import Page +from typing import Tuple + +from .dash_do_base import DashboardRetrieveIncidentAndDoInfeasibleTask, DashDoFinalTask +from .utils.infeasible_configs import get_infeasible_form_config + +from ..base import AbstractServiceNowTask +from ..dashboard import ReportMinMaxRetrievalTask + +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.form import CreateItemRequestTask + + +class DashboardRetrieveIncidentAndRequestItemInfeasibleTask( + DashboardRetrieveIncidentAndDoInfeasibleTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + item: str = None, + question: str = None, + dashboard_class: AbstractServiceNowTask = None, + provide_reason: bool = None, + ) -> None: + """ + Retrieve the best performing agent and request an item for them. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. ""Order an item for the best performing employee from the list" + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + question=question, + dashboard_class=dashboard_class, + function=get_infeasible_form_config, + provide_reason=provide_reason, + ) + if not item: + raise Exception("No item passed to assign") + self.item = item + self.task_description = "" + self.short_description = ( + f"Order an {self.item} for the best performing employee from the list." + ) + self.attribute_name = "assigned_to" # Return full name + self.filter_than = "greater" + self.prefix = "IRI" + + def set_compositional_task(self) -> None: + # The unique name for the user is created once the task is instantiated + requested_items = table_api_call( + instance=self.instance, + table="sc_req_item", + method="GET", + )["result"] + current_requested_items_numbers = [ + requested_item["number"] for requested_item in requested_items + ] + + agent_full_names, agent_value_sysids = self.get_agent_values( + self.attribute_name, self.filter_than + ) + self.agent_value_sysids = agent_value_sysids + + requested_item_numbers = [] + + for _ in range(len(agent_full_names)): + requested_item_number = "RITM" + str(random.randint(1000000, 9999999)) + while ( + requested_item_number in current_requested_items_numbers + or requested_item_number in requested_item_numbers + ): + requested_item_number = "RITM" + str(random.randint(1000000, 9999999)) + requested_item_numbers.append(requested_item_number) + + self.requested_item_numbers = requested_item_numbers + + create_item_request_subtasks = [] + + for agent_full_name, requested_item_number in zip(agent_full_names, requested_item_numbers): + request_item_config = { + "fields": { + "number": "Number", + "cat_item": "Item", + "requested_for": "Requested for", + "quantity": "Quantity", + }, + "task_fields": ["number", "cat_item", "requested_for", "quantity"], + "template_record": { + "number": requested_item_number, + "cat_item": self.item, + "requested_for": agent_full_name, + "quantity": "1", + }, + "infeasible_task_fields": ["number", "cat_item", "quantity"], + } + request_item_config, self.infeasible_reasons = self.function( + config=request_item_config, random=self.random + ) + create_item_request_subtask = [ + # Navigate to the item request list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Open Records", + "module": "Open Records > Items", + "url": "/now/nav/ui/classic/params/target/sc_req_item_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create an item request + CreateItemRequestTask( + instance=self.instance, + fixed_config=request_item_config, + is_validated=False, + used_in_level_2=True, + check_record_created=False, + ), + ] + create_item_request_subtasks += create_item_request_subtask + + self.compositional_task = create_item_request_subtasks + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.create_report() + self.set_compositional_task() + config = self.fixed_config if self.fixed_config else self._get_config() + if self.level == 3: + self.task_description = ( + self.task_description + + f"Value to retrieve: {self.description_mapping[self.question]} of all the incidents. Comparator: Greather than or equal to the value.\n" + + f"Task: Request items with the following information: \n" + + f"Item: {self.item}, Quantity: 1.\n" + + f"Request the item for each of the agents mentioned above. You can use the item numbers: {self.requested_item_numbers}, one for each request." + ) + + goal, info = super().setup_goal(page=page, config=config) + + if self.level == 2: + goal = ( + self.task_description + + f"1. Navigate to the CMDB reports and look for the report with the mentioned hashtag. \n" + + f"2. Find the agents with number of incidents greater than or equal to the {self.description_mapping[self.question]} of the incidents assigned to every one. \n" + + f"3. Navigate to Open Records > Items. \n" + + f"4. Create new item requests with the following field values:- 'Item: {self.item}, Quantity: 1' and assign them to each of the agents. You will create as many item requests as there are agents.\n" + + f"You should use the following request numbers for each item request (one for each): {self.requested_item_numbers}." + ) + + return goal, info + + def teardown(self) -> None: + for requested_item_number in self.requested_item_numbers: + created_item_request_response = table_api_call( + instance=self.instance, + table="sc_req_item", + params={ + "sysparm_query": f"number={requested_item_number}", + }, + method="GET", + )["result"] + if len(created_item_request_response) > 1: + raise Exception("Multiple request items created") + if len(created_item_request_response) == 1: + created_item_request_sysid = created_item_request_response[0]["sys_id"] + db_delete_from_table( + instance=self.instance, + table="sc_req_item", + sys_id=created_item_request_sysid, + ) + return super().teardown() + + +class DashboardRetrieveIncidentAndMaxRequestAppleWatchInfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleWatchInfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleWatch2InfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch Series 2", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleWatch2InfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple Watch Series 2", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIpad3InfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPad 3", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIpad3InfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPad 3", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIphone13proInfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13 pro", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIphone13proInfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13 pro", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIphone13InfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestAppleIphone13InfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Apple iPhone 13", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestGalaxyNote20InfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Galaxy Note 20", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestGalaxyNote20InfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Galaxy Note 20", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestGoogleNexus7InfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Google Nexus 7", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestGoogleNexus7InfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Google Nexus 7", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestMicrosoftSurfacePro3InfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Microsoft Surface Pro 3", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestMicrosoftSurfacePro3InfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Microsoft Surface Pro 3", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestPixel4aInfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Pixel 4a", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestPixel4aInfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Pixel 4a", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +class DashboardRetrieveIncidentAndMaxRequestWindowsSurfacePro4InfeasibleWithReasonTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Windows Surface Pro 4", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=True, + ) + + +class DashboardRetrieveIncidentAndMaxRequestWindowsSurfacePro4InfeasibleTask( + DashboardRetrieveIncidentAndRequestItemInfeasibleTask, DashDoFinalTask +): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Retrieve the best performing agent and request an apple watch for them. + """ + super().__init__( + instance=instance, + seed=seed, + fixed_config=fixed_config, + level=level, + item="Windows Surface Pro 4", + question="max", + dashboard_class=ReportMinMaxRetrievalTask, + provide_reason=False, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, DashDoFinalTask) and var is not DashDoFinalTask +] diff --git a/src/browsergym/workarena/tasks/compositional/delete_record.py b/src/browsergym/workarena/tasks/compositional/delete_record.py new file mode 100644 index 0000000..edc781b --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/delete_record.py @@ -0,0 +1,341 @@ +import faker + +faker = faker.Faker() +import json + +from playwright.sync_api import Page +from typing import List, Tuple + +from .base import AbstractServiceNowTask + +from ..utils.utils import check_url_suffix_match + +from ...api.utils import db_delete_from_table, table_api_call + + +class DeleteRecordTask(AbstractServiceNowTask): + """ + Delete a record from a list. + + Parameters: + ----------- + instance: SNowInstance + The instance to use. + start_rel_url: str + The relative URL of the list containing the record to delete. + list_name: str + The displayed name of the list containing the record to delete. + fixed_config: dict + Configuration to use for the task. If provided, the task will use the provided configuration instead of + selecting a random one. See browsergym/workarena/data_files/task_configs/filter_change_request_list_task.json + for an example of a configuration file. + all_configs: list[dict] + A list of all possible configurations to use for the task. + record_sys_id: str + The sys_id of the record to delete. If not provided, a record will be created during the setup. + record_number: str + The number of the record to delete; used in the cheat. If not provided, the cheat will select the last one. + """ + + def __init__( + self, + seed: int = None, + instance=None, + start_rel_url: str = "", + list_name: str = "", + fixed_config: dict = None, + all_configs: list[dict] = None, + record_sys_id: str = None, + record_number: str = None, + **kwargs, + ) -> None: + super().__init__(seed=seed, instance=instance, start_rel_url=start_rel_url) + self.list_name = list_name + self.table_name = start_rel_url.split("/")[-1].split("_list.do")[0] + self.fixed_config = fixed_config + self.config = None + self.pretty_printed_field_name = None + self.field_name = None + self.field_value = None + self.other_fields = None + self.all_configs = all_configs + # If the record_sys_id is not provided, it will be created during the setup + self.record_sys_id = record_sys_id + self.record_number = record_number + self.__dict__.update(kwargs) + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.config = ( + self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) + ) + self.field_name = self.config.get("field_name") + self.pretty_printed_field_name = self.config.get("pretty_printed_field_name") + self.field_value = self.config.get("field_value") + self.other_fields = self.config.get("other_fields") + if self.record_sys_id is None: + # First, check if the record already exists + record = table_api_call( + instance=self.instance, + table=self.table_name, + params={ + "sysparm_query": f"{self.field_name}={self.field_value}", + "sysparm_fields": "sys_id", + }, + )["result"] + if len(record) > 0: + raise ValueError( + f"Record already with {self.field_name} = {self.field_value} exists. Please delete it before proceeding." + ) + + self.record_sys_id = table_api_call( + instance=self.instance, + table=self.table_name, + data=json.dumps( + { + self.field_name: self.field_value, + **self.other_fields, + } + ), + method="POST", + )["result"]["sys_id"] + + goal = self.get_pretty_printed_description() + + return goal, {} + + def get_init_scripts(self) -> List[str]: + return super().get_init_scripts() + ["registerGsftMainLoaded();"] + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + task_info = f"- Delete the record with {self.pretty_printed_field_name}={self.field_value} from the {self.list_name} list." + + return task_info + + def cheat(self, page: Page, chat_messages: list[str]) -> None: + super().cheat(page, chat_messages) + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + + # If the record number is provided, click on the record with that number... + if self.record_number is not None: + frame.locator(f"[aria-label='Preview record: {self.record_number}']").click() + page.wait_for_timeout(500) + frame.get_by_text("Open Record").click() + # ....Otherwise, otherwise filter the list and click on the record + else: + # Search for the record + frame.get_by_label( + f"Search a specific field of the {self.list_name} list" + ).select_option(f"{self.field_name}") + search_input = frame.locator('input[aria-label="Search"]') + search_input.click() + search_input.fill(self.field_value) + search_input.press("Enter") + page.wait_for_function( + "typeof window.gsft_main !== 'undefined' && window.gsft_main.WORKARENA_LOAD_COMPLETE" + ) + # Click on the record to open it + # The first 2 displays of the record are in the search bar; the 3rd and last will be the link to open it + frame.get_by_label(self.field_value).last.click() + + page.wait_for_function( + "typeof window.gsft_main !== 'undefined' && window.gsft_main.WORKARENA_LOAD_COMPLETE" + ) + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + # Click on delete, then confirm delete in the popup + frame.get_by_text("delete").first.click() + frame.wait_for_selector('header[aria-label="Confirmation"]') + page.keyboard.press("Enter") + # Wait for record to be updated in the DB + record_deleted = False + while not record_deleted: + record = table_api_call( + instance=self.instance, + table=self.table_name, + params={ + "sysparm_query": f"{self.field_name}={self.field_value}", + "sysparm_fields": "sys_id", + }, + )["result"] + record_deleted = len(record) == 0 + page.wait_for_timeout(3000) + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + """ + Validate the solution + """ + record = table_api_call( + instance=self.instance, + table=self.table_name, + params={"sysparm_query": f"{self.field_name}={self.field_value}"}, + )["result"] + if len(record) > 0: + return 0, False, "", {"message": "Record was not deleted."} + + return 1, True, "Nice work, thank you!", {"message": "Record was deleted successfully."} + + def teardown(self) -> None: + super().teardown() + result = table_api_call( + instance=self.instance, + table=self.table_name, + params={ + "sysparm_query": f"{self.field_name}={self.field_value}", + "sysparm_fields": "sys_id", + }, + ) + if len(result["result"]) > 0: + db_delete_from_table( + instance=self.instance, + table=self.table_name, + sys_id=self.record_sys_id, + ) + + +class DeleteUserTask(DeleteRecordTask): + def __init__(self, instance=None, fixed_config=None, record_sys_id=None, **kwargs) -> None: + super().__init__( + instance=instance, + start_rel_url="/now/nav/ui/classic/params/target/sys_user_list.do", + list_name="Users", + fixed_config=fixed_config, + record_sys_id=record_sys_id, + **kwargs, + ) + if fixed_config is None: + first_name = faker.first_name() + last_name = faker.last_name() + email = first_name.lower() + "." + last_name.lower() + "@workarena.com" + self.fixed_config = { + "field_name": "user_name", + "pretty_printed_field_name": "User ID", + "field_value": first_name + " " + last_name, + "other_fields": {"email": email}, + } + + +class DeleteExpenseLineExpenseManagementTask(DeleteRecordTask): + """ + Delete one row from the expense lines list + + Args: + -------- + goal_type (str): + The type of goal to generate. Choice of "base", "date", "amount", "any" + level (int): + The level of the task + skip_description (bool): + Whether to skip the description of the task + + """ + + def __init__( + self, + instance=None, + fixed_config=None, + record_sys_id=None, + goal_type="base", + level=2, + skip_description=False, + **kwargs, + ) -> None: + super().__init__( + instance=instance, + start_rel_url="/now/nav/ui/classic/params/target/fm_expense_line_list.do", + list_name="Expense Lines", + fixed_config=fixed_config, + record_sys_id=record_sys_id, + **kwargs, + ) + self.goal_type = goal_type + self.level = level + self.skip_description = skip_description + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in compositional tasks. + called by subclasses + """ + task_info = f"Delete expense lines with duplicated short descriptions" + if self.skip_description: + task_info = "" + elif self.level == 3: + task_info += f" according to the protocol." + elif self.goal_type == "base": + task_info += f" where the duplicated expense lines are not associated with tasks." + elif self.goal_type == "date": + task_info += f", keeping only the one that has the oldest date." + elif self.goal_type == "amount": + task_info += f", keeping only the most expensive duplicate." + elif self.goal_type == "any": + task_info += f", keeping only one." + + return task_info + + +class DeleteExpenseLineKnapsack(DeleteRecordTask): + """ + Delete one row from the expense lines list + + Args: + -------- + goal_type (str): + The type of goal to generate. Choice of "base", "date", "amount", "any" + answer_format (str): + The type of answer to generate. Choice of total_return_only, total_return_and_investments, investments_only, cleanup, cleanup_and_return + level (int): + The level of the task + skip_description (bool): + Whether to skip the description of the task + + """ + + def __init__( + self, + instance=None, + fixed_config=None, + record_sys_id=None, + goal_type="base", + level=2, + answer_format=None, + skip_description=False, + **kwargs, + ) -> None: + super().__init__( + instance=instance, + start_rel_url="/now/nav/ui/classic/params/target/fm_expense_line_list.do", + list_name="Expense Lines", + fixed_config=fixed_config, + record_sys_id=record_sys_id, + **kwargs, + ) + self.goal_type = goal_type + self.level = level + self.answer_format = answer_format + self.skip_description = skip_description + + def get_pretty_printed_description(self) -> str: + if self.skip_description: + return "" + if self.level == 3: + task_info = "Allocate the budget to maximize revenue." + elif self.level == 2: + task_info = f"Allocate the budget to maximize revenue. This involves going over expense lines and identifying the ones maximizing revenue while fitting in the allowed budget of {self.budget}. The returns are written in their short description." + if self.answer_format == "total_return_only": + task_info += " Provide only the total return of the investments in the chat." + if self.answer_format == "total_return_and_investments": + task_info += " Provide the total return of the investments as well as the number of the investments in the chat." + if self.answer_format == "investments_only": + task_info += " Provide only the numbers of the investments in the chat." + if self.answer_format == "cleanup": + task_info += " Delete the investments that will not be kept so that only the selected investments remain." + if self.answer_format == "cleanup_and_return": + task_info += " Delete the investments that will not be kept so that only the selected investments remain as well as returning their total value in the chat." + + return task_info + + +__TASKS__ = [DeleteUserTask] diff --git a/src/browsergym/workarena/tasks/compositional/edit_knowledge_base.py b/src/browsergym/workarena/tasks/compositional/edit_knowledge_base.py new file mode 100644 index 0000000..a5e434d --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/edit_knowledge_base.py @@ -0,0 +1,457 @@ +import html +import json +import random + +from faker import Faker +from playwright.sync_api._generated import Page +from typing import Tuple + +fake = Faker() + +from ...api.knowledge import give_kb_read_permissions +from ...api.utils import table_api_call +from ..base import AbstractServiceNowTask +from .base import CompositionalTask +from ...config import KB_FILEPATH, PROTOCOL_KB_NAME +from ...instance import SNowInstance +from ..knowledge import KnowledgeBaseSearchTask, AddCommentToKnowledgeArticleTask +from ..navigation import AllMenuTask + + +class EditKnowledgeBaseTask(CompositionalTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 3, + ) -> None: + """ + Create a compositional task with specific subtasks + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of tuples, each containing a subtask, its configuration and whether or not it should be validated. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. e.g. "Referring to company protocol 'Edit a knowledge article', edit the knowledge base to handle the incorrect information: \n' + short_description: str + A short description of the task to be completed. e.g. "Edit knowledge base entries for address of parking lot." + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + self.protocol_name = "Edit a knowledge article to manage incorrect information" + + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + protocol_name=self.protocol_name, + user_roles=["itil"], # Required permission to access service desk for l3 + ) + with open(KB_FILEPATH, "r") as f: + self.kb_entries = json.load(f) + if not hasattr(self, "_base_initial_instance"): + self._base_initial_instance = self.instance + self.adhoc_kb_name = None + self.task_description = None + self.short_description = None + + def create_adhoc_kb(self): + user_full_name = " ".join(self._base_user_name.split(".")[:-1]) + adhoc_kb_name = f"{user_full_name}'s Knowledge Base" + self.adhoc_kb_name = adhoc_kb_name + + kb = table_api_call( + instance=self._base_initial_instance, + table="kb_knowledge_base", + method="POST", + data=json.dumps( + { + "title": self.adhoc_kb_name, + } + ), + )["result"] + + return kb["sys_id"] + + def get_random_article_name(self): + kb = table_api_call( + instance=self.instance, + table="kb_knowledge", + params={ + "sysparm_query": f"kb_knowledge_base={self.adhoc_kb_sys_id}", + "sysparm_fields": "short_description", + }, + )["result"] + self.article_titles = [kb_article["short_description"] for kb_article in kb] + + article_date = fake.date_this_year().strftime("%Y-%m-%d") + base_article_name = self.base_config["item"].capitalize() + article_title = f"{base_article_name}-{article_date}" + while article_title in self.article_titles: + article_date = fake.date_this_year().strftime("%Y-%m-%d") + article_title = f"{base_article_name}-{article_date}" + + return article_title + + def create_article(self, article_name, article_text): + if article_name in self.article_titles: + raise Exception("Article with the name already exists...") + + adhoc_kb_response = table_api_call( + instance=self._base_initial_instance, # admin permissions to contribute to the KB + table="kb_knowledge", + method="POST", + data=json.dumps( + { + "short_description": article_name, + "sys_class_name": "kb_knowledge", + "text": article_text, + "article_type": "text", + "kb_knowledge_base": self.adhoc_kb_sys_id, + } + ), + )["result"] + + return adhoc_kb_response + + def setup_goal(self, page: Page) -> tuple[str, dict]: + # Create the KB + self.adhoc_kb_sys_id = self.create_adhoc_kb() + # Sample a configuration + self.base_config = self.random.choice(self.kb_entries) + self.incorrect_kb_article_name = self.get_random_article_name() + self.correct_kb_article_name = self.get_random_article_name() + self.item = self.base_config["item"] + self.correct_answer = self.base_config["value"] + + self.incorrect_answer = " ".join( + [fake.word() for _ in range(len(self.correct_answer.split()))] + ) # Random incorrect answer with the same number of words as the correct answer + + incorrect_kb_article = self.create_article( + self.incorrect_kb_article_name, + self.base_config["article"].replace(self.correct_answer, self.incorrect_answer), + ) + self.incorrect_kb_article_sys_id = incorrect_kb_article["sys_id"] + self.incorrect_kb_article_number = incorrect_kb_article["number"] + + correct_kb_article = self.create_article( + self.correct_kb_article_name, self.base_config["article"] + ) + self.correct_kb_article_sys_id = correct_kb_article["sys_id"] + self.correct_kb_article_number = correct_kb_article["number"] + + config = self.fixed_config if self.fixed_config else self._get_config() + + give_kb_read_permissions( + self._base_initial_instance, + self._base_user_sysid, + self._base_user_name, + self.adhoc_kb_sys_id, + self.adhoc_kb_name, + ) + + if self.level == 3: + protocol_kb_sys_id = table_api_call( + instance=self._base_initial_instance, + table="kb_knowledge_base", + params={"sysparm_query": f"title={PROTOCOL_KB_NAME}"}, + )["result"][0]["sys_id"] + give_kb_read_permissions( + self._base_initial_instance, + self._base_user_sysid, + self._base_user_name, + protocol_kb_sys_id, + PROTOCOL_KB_NAME, + ) + + # Get the task description + self.short_description = f"Edit knowledge base article for {self.item}" + self.task_description = ( + f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) edit the knowledge base to handle incorrect information. \n' + + f'Searching for "{self.item}" in the knowledge base gives different articles as the output: "{self.incorrect_kb_article_name}" with number "{self.incorrect_kb_article_number}" and "{self.correct_kb_article_name}" with number "{self.correct_kb_article_number}". \n' + # + f'One of the articles has incorrect information "{self.incorrect_answer}" and the other one has the correct answer "{self.correct_answer}". \n' + + f'The correct information for "{self.item}" should be {self.correct_answer}. ' + ) + + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[tuple[AbstractServiceNowTask, dict, bool]]: + """Add more extensive definition here.""" + + self.incorrect_config = { + "item": self.item, + "kb_article_title": self.incorrect_kb_article_name, + "value": self.incorrect_answer, + "question": self.base_config["questions"][0], + # "replaced_text": self.incorrect_answer, + "comment": f"This article has incorrect information and is obsolete. Please refer to the article numbered {self.correct_kb_article_number} for reference.", + "alternative_answers": [ + self.incorrect_answer, + ], + } + + self.correct_config = { + "item": self.item, + "kb_article_title": self.correct_kb_article_name, + "value": self.correct_answer, + "question": self.base_config["questions"][0], + "comment": f"This article has correct information. Please DO NOT refer to the article numbered {self.incorrect_kb_article_number} for reference.", + "alternative_answers": [ + self.correct_answer, + ], + } + + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": 'Can you find the "Edit Knowledge Article Protocol" in the Knowledge Base?', + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + + search_and_comment_knowledge_base_incorrect_subtask = [ + # Navigate to the knowledge base home page + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=True, + ), + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config=self.incorrect_config, + is_validated=False, + used_in_level_2=True, + is_correct=False, + is_only_navigating=True, + search_by_title=True, + ), + # Search the knowledge base for the incorrect article + AddCommentToKnowledgeArticleTask( + instance=self.instance, + fixed_config=self.incorrect_config, + is_validated=False, + used_in_level_2=True, + ), + ] + + search_and_comment_knowledge_base_correct_subtask = [ + # Navigate to the knowledge base home page + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=True, + ), + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config=self.correct_config, + is_validated=False, + used_in_level_2=True, + is_only_navigating=True, + search_by_title=True, + ), + # Search the knowledge base for the incorrect article + AddCommentToKnowledgeArticleTask( + instance=self.instance, + fixed_config=self.correct_config, + is_validated=False, + used_in_level_2=True, + ), + ] + + config = ( + navigate_to_protocol_subtask + + search_and_comment_knowledge_base_incorrect_subtask + + search_and_comment_knowledge_base_correct_subtask + ) + + return config + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + incorrect_article_kb_sys_id = table_api_call( + instance=self.instance, + table="kb_knowledge", + params={ + "sysparm_query": f"short_description={self.incorrect_kb_article_name}", + }, + )["result"][0]["sys_id"] + + incorrect_synonyms = [ + "incorrect", + "wrong", + "false", + "inaccurate", + "mistaken", + "erroneous", + "improper", + "invalid", + "untrue", + "misleading", + "off", + "obsolete", + ] + incorrect_validated = 0 + all_comments = table_api_call( + instance=self.instance, + table="kb_feedback", + params={ + "sysparm_query": f"article={incorrect_article_kb_sys_id}", + }, + )["result"] + + for comment in all_comments: + if ( + any( + incorrect_synonym.lower() in html.unescape(comment["comments"]).lower() + for incorrect_synonym in incorrect_synonyms + ) + and comment["sys_created_by"] == self._base_user_name + and self.correct_kb_article_number.lower() + in html.unescape(comment["comments"]).lower() + ): + incorrect_validated = 1 + break + + correct_article_kb_sys_id = table_api_call( + instance=self.instance, + table="kb_knowledge", + params={ + "sysparm_query": f"short_description={self.correct_kb_article_name}", + }, + )["result"][0]["sys_id"] + + correct_synonyms = [ + "correct", + "right", + "accurate", + "true", + "exact", + "precise", + "proper", + "valid", + "factual", + "appropriate", + "verifiable", + "up-to-date", + "up to date", + ] + correct_validated = 0 + all_comments = table_api_call( + instance=self.instance, + table="kb_feedback", + params={ + "sysparm_query": f"article={correct_article_kb_sys_id}", + }, + )["result"] + + for comment in all_comments: + if ( + any( + correct_synonym.lower() in html.unescape(comment["comments"]).lower() + for correct_synonym in correct_synonyms + ) + and comment["sys_created_by"] == self._base_user_name + and self.incorrect_kb_article_number.lower() + in html.unescape(comment["comments"]).lower() + ): + correct_validated = 1 + break + + if incorrect_validated and correct_validated: + # Validate final_l3 tasks + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + elif incorrect_validated and not correct_validated: + return ( + 0, + False, + "", + { + "message": "Comment successfully added to the incorrect article but not the correct article." + }, + ) + elif not incorrect_validated and correct_validated: + return ( + 0, + False, + "", + { + "message": "Comment successfully added to the correct article but not the incorrect article." + }, + ) + else: + return ( + 0, + False, + "", + { + "message": "Comment not added to either the correct article or the incorrect article." + }, + ) + + def teardown(self) -> None: + # Delete created articles + table_api_call( + instance=self._base_initial_instance, + table=f"kb_knowledge/{self.incorrect_kb_article_sys_id}", + method="DELETE", + ) + table_api_call( + instance=self._base_initial_instance, + table=f"kb_knowledge/{self.correct_kb_article_sys_id}", + method="DELETE", + ) + + # Archive knowledge base + table_api_call( + instance=self._base_initial_instance, + table=f"kb_knowledge_base/{self.adhoc_kb_sys_id}", + method="PATCH", + json={"title": f"archived_{self.adhoc_kb_sys_id}", "active": "false"}, + ) + return super().teardown() + + +__TASKS__ = [EditKnowledgeBaseTask] diff --git a/src/browsergym/workarena/tasks/compositional/expense_management.py b/src/browsergym/workarena/tasks/compositional/expense_management.py new file mode 100644 index 0000000..7540552 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/expense_management.py @@ -0,0 +1,598 @@ +import re + +from datetime import timedelta +from faker import Faker +from typing import List, Tuple + +fake = Faker() + +from playwright.sync_api._generated import Page + +from .base import HumanEvalTask +from .delete_record import DeleteExpenseLineExpenseManagementTask +from .filter_and_do import FilterAndDoTask + +from ..base import AbstractServiceNowTask + +from ...api.change_request import create_change_request +from ...api.expense_line import create_expense_line +from ...api.utils import table_api_call, db_delete_from_table +from ...config import ( + # Expected columns for the different lists + EXPECTED_EXPENSE_LINE_COLUMNS_PATH, +) +from ...instance import SNowInstance + + +class ExpenseManagementTask(FilterAndDoTask): + """Task to manage expenses. + Args: + + num_duplicates: int + The number of duplicate expenses to create + extra_expenses: int + The number of extra expenses to create (total expenses will be num_duplicates + extra_expenses) + goal_type: str + The type of goal to generate. Choice of "base", "date", "amount", "any". + - "base": one expense is linked to a change request, others are not and are expected to be deleted + - "date": none of the expenses are linked to change requests; the oldest one is expeted to be deleted + - "amount": none of the expenses are linked to change requests and they are all created on the same date; + the most expensive one is expeted to be deleted + - "any": any of the expenses can be deleted + """ + + min_allowed_amount = 100 + max_allowed_amount = 10000 + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + num_duplicates: int = 2, + extra_expenses: int = 2, + goal_type: str = "base", + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "module": "Expense Lines", + "application": "Cost", + }, + level=level, + protocol_name="Managing Your Existing Expenses", + ) + self.num_duplicates = num_duplicates + self.extra_expenses = extra_expenses + self.total_expenses = num_duplicates + extra_expenses + self.goal_type = goal_type + self.change_request_sysids = [] + + # mappings between number -> (is_duplicate, sys_id) + self.expense_lines = {} + self.expense_to_keep_number = None # The number of the expense that will be kept; i.e. not deleted by the cheat/agent if successful + + self.expense_hashtag = "#SERIES-" + self.unique_id[:10] + self.short_description = f"Managing Your Existing Expenses" + self.task_description = f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) manage your expenses with short description containing hashtag {self.expense_hashtag}. ' + self.tasks = [] + + def _setup_list(self) -> None: + """The setup might be a bit complex on a first read. To understand it better, refer to the protocol for this task.""" + self.filter_config = { + "list_url": "/now/nav/ui/classic/params/target/fm_expense_line_list.do", + "expected_fields_path": EXPECTED_EXPENSE_LINE_COLUMNS_PATH, + "filter_columns": [ + "short_description", + ], + "filter_kind": "AND", + "filter_operators": ["contains"], + "filter_values": [ + f"{self.expense_hashtag}", + ], + } + + # Short description to use for duplicate expenses + duplicate_short_description = f"{fake.sentence(4)}" + + # Set the default amount and date in case uniform_amount and uniform_date are True + amount = round(self.random.uniform(self.min_allowed_amount, self.max_allowed_amount), 2) + start_date = fake.date_this_decade(before_today=True, after_today=False) + date = start_date + + most_expensive_amount = float("-inf") + + most_expensive_duplicate_expense_number = None + oldest_duplicate_expense_number = None + only_expense_with_change_request = None + # id of expenses will be this id + their order in the creation + unique_id = str(int(self.unique_id.replace("-", ""), 16))[:10] + for i in range(self.total_expenses): + expense_number = f"EXP-{i}{unique_id }" + is_duplicate = i < self.num_duplicates + # set task sys_id to empty string if no change request is created + task_sys_id = "" + + # Set a random date between start_date and today + if self.goal_type in ["base", "date"] and i > 0: + date = str( + fake.date_between(start_date=start_date + timedelta(1), end_date="today") + ) + else: + date = start_date + if i == 0: + oldest_duplicate_expense_number = expense_number + + # In the 'any' case, there are no change requests, all dates are the same and the prices are the same + if self.goal_type != "any": + amount = round( + self.random.uniform(self.min_allowed_amount, self.max_allowed_amount), 2 + ) + if is_duplicate and amount > most_expensive_amount: + most_expensive_amount = amount + most_expensive_duplicate_expense_number = expense_number + + # Create a change request for the base case + if self.goal_type == "base" and i == 0: + task_sys_id, _ = create_change_request( + instance=self.instance, + user_sys_id=self._base_user_sysid, + hashtag=self.expense_hashtag, + impact=2, + risk=2, + random=self.random, + ) + self.change_request_sysids.append(task_sys_id) + only_expense_with_change_request = expense_number + + # Set the short description for the duplicate expenses; otherwise pass None, which will generate a random one + short_description = duplicate_short_description if i < self.num_duplicates else None + + expense_sys_id, _ = create_expense_line( + instance=self.instance, + amount=amount, + number=expense_number, + date=str(date), + short_description=short_description, + expense_hashtag=self.expense_hashtag, + user_sys_id=self._base_user_sysid, + task_sys_id=task_sys_id, + ) + self.expense_lines[expense_number] = (is_duplicate, expense_sys_id) + + # keep the number of the expense that will be linked to the change request + if self.goal_type == "base": + self.expense_to_keep_number = only_expense_with_change_request + # keep the oldest expense + elif self.goal_type == "date": + self.expense_to_keep_number = oldest_duplicate_expense_number + elif self.goal_type == "amount": + self.expense_to_keep_number = most_expensive_duplicate_expense_number + else: + self.expense_to_keep_number = oldest_duplicate_expense_number + + # As the task description redundant, we keep only the first one and skip the rest + skip_description = False + # Create the tasks to delete the extra expenses + for expense_number, (is_duplicate, expense_sys_id) in self.expense_lines.items(): + if expense_number == self.expense_to_keep_number or not is_duplicate: + continue + self.tasks.append( + DeleteExpenseLineExpenseManagementTask( + instance=self.instance, + fixed_config={ + "field_name": "number", + "field_value": f"{expense_number}", + }, + is_validated=False, + used_in_level_2=True, + record_sys_id=expense_sys_id, + record_number=expense_number, + level=self.level, + skip_description=skip_description, + goal_type=self.goal_type, + ) + ) + skip_description = True + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + expenses = table_api_call( + instance=self.instance, + table="fm_expense_line", + params={ + "sysparm_query": f"short_descriptionLIKE{self.expense_hashtag}", + "sysparm_fields": "number,amount,sys_id", + }, + )["result"] + # There should remain only one duplicate expense after the task is completed and the extra expenses should reamin + target_num_expenses = self.extra_expenses + 1 + # Check that only one of the duplicated expenses exists and it is the right one + if len(expenses) != target_num_expenses: + return ( + 0, + False, + "", + {"message": "Wrong number of expenses."}, + ) + + existing_expense_numbers = {expense["number"] for expense in expenses} + for expense_number, (is_duplicate, _) in self.expense_lines.items(): + # Check that only one of the duplicated expenses exists and it is the right one + if expense_number == self.expense_to_keep_number: + if expense_number not in existing_expense_numbers: + return ( + 0, + False, + "", + {"message": "The expected duplicate to keep is missing."}, + ) + + # Check that other duplicates have been deleted + elif is_duplicate and expense_number in existing_expense_numbers: + return ( + 0, + False, + "", + {"message": "An unexpected duplicate is present."}, + ) + # Check that the extra expenses have not been deleted + elif not is_duplicate and expense_number not in existing_expense_numbers: + return ( + 0, + False, + "", + {"message": "An extra expense has been deleted."}, + ) + + # Validate final_l3 tasks + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + + def teardown(self) -> None: + for _, expense_sys_id in self.expense_lines.values(): + record_exists = table_api_call( + instance=self.instance, + table="fm_expense_line", + params={"sysparm_query": f"sys_id={expense_sys_id}"}, + )["result"] + if not record_exists: + continue + db_delete_from_table( + instance=self.instance, + table="fm_expense_line", + sys_id=expense_sys_id, + ) + for change_request_sys_id in self.change_request_sysids: + record_exists = table_api_call( + instance=self.instance, + table="change_request", + params={"sysparm_query": f"sys_id={change_request_sys_id}"}, + )["result"] + if not record_exists: + continue + db_delete_from_table( + instance=self.instance, + table="change_request", + sys_id=change_request_sys_id, + ) + super().teardown() + + +class BasicExpenseManagementSmallTask(ExpenseManagementTask, HumanEvalTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 2, + extra_expenses: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="base", + level=level, + ) + + +class DateBasedExpenseManagementSmallTask(ExpenseManagementTask, HumanEvalTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 2, + extra_expenses: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="date", + level=level, + ) + + +class AmountBasedExpenseManagementSmallTask(ExpenseManagementTask, HumanEvalTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 2, + extra_expenses: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="amount", + level=level, + ) + + +class EasyExpenseManagementTask(ExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 2, + extra_expenses: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="any", + level=level, + ) + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + expenses = table_api_call( + instance=self.instance, + table="fm_expense_line", + params={ + "sysparm_query": f"short_descriptionLIKE{self.expense_hashtag}", + "sysparm_fields": "number,amount,sys_id", + }, + )["result"] + # There should remain only one expense after the task is completed and the extra expenses should reamin + target_num_expenses = self.extra_expenses + 1 + # Check that only one of the duplicated expenses exists and it is the right one + if len(expenses) != target_num_expenses: + return ( + 0, + False, + "", + {"message": "Wrong number of expenses."}, + ) + + existing_expense_numbers = {expense["number"] for expense in expenses} + for expense_number, (is_duplicate, _) in self.expense_lines.items(): + if not is_duplicate and expense_number not in existing_expense_numbers: + return ( + 0, + False, + "", + {"message": "An extra expense has been deleted."}, + ) + + # Validate final_l3 tasks + reward, done, message, info = FilterAndDoTask.validate(self, page, chat_messages) + return reward, done, message, info + + +class EasyExpenseManagementSmallTask(EasyExpenseManagementTask, HumanEvalTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 2, + extra_expenses: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + level=level, + ) + + +class BasicExpenseManagementMediumTask(ExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 4, + extra_expenses: int = 4, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="base", + level=level, + ) + + +class DateBasedExpenseManagementMediumTask(ExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 4, + extra_expenses: int = 4, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="date", + level=level, + ) + + +class AmountBasedExpenseManagementMediumTask(ExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 4, + extra_expenses: int = 4, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="amount", + level=level, + ) + + +class EasyExpenseManagementMediumTask(EasyExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 4, + extra_expenses: int = 4, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + level=level, + ) + + +class BasicExpenseManagementLargeTask(ExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 6, + extra_expenses: int = 6, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="base", + level=level, + ) + + +class DateBasedExpenseManagementLargeTask(ExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 6, + extra_expenses: int = 6, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="date", + level=level, + ) + + +class AmountBasedExpenseManagementLargeTask(ExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 6, + extra_expenses: int = 6, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + goal_type="amount", + level=level, + ) + + +class EasyExpenseManagementLargeTask(EasyExpenseManagementTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_duplicates: int = 6, + extra_expenses: int = 6, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_duplicates=num_duplicates, + extra_expenses=extra_expenses, + level=level, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) + and issubclass(var, FilterAndDoTask) + and var is not FilterAndDoTask + and var is not ExpenseManagementTask + and var is not EasyExpenseManagementTask +] diff --git a/src/browsergym/workarena/tasks/compositional/filter_and_do.py b/src/browsergym/workarena/tasks/compositional/filter_and_do.py new file mode 100644 index 0000000..2662f27 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/filter_and_do.py @@ -0,0 +1,139 @@ +from faker import Faker + +fake = Faker() + +from playwright.sync_api._generated import Page + +from browsergym.workarena.tasks.knowledge import KnowledgeBaseSearchTask +from browsergym.workarena.tasks.list import FilterListTask +from browsergym.workarena.tasks.navigation import AllMenuTask + +from .base import CompositionalTask + +from ..base import AbstractServiceNowTask + +from ...instance import SNowInstance + + +class FilterAndDoTask(CompositionalTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + navigation_config: dict = None, + protocol_name: str = None, + level: int = 2, + ) -> None: + """ + Generic task to navigate to a specific page, run a filter and perform a task. + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of tuples, each containing a subtask + navigation_config: dict + Configuration to use for the navigation to the list that will be filtered. Contains application and module. + URL is not necessary as the navigation steps are not validated + protocol_name: str + The name of the protocol to refer to in the task description for L3. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + + Attributes: + ----------- + filter_config: dict + Configuration to use for the filter that will be applied to the list. Contains filter_columns, filter_values and filter_kind in addition to the path to the expected fields in the list. + this is set by the _setup_list method. + tasks: List[AbstractServiceNowTask] + The tasks to perform after having filtered the list. Set by the child setup + task_description: str + The start of the task description to be completed. Provided by the child class. + short_description: str + A short description of the task to be completed. "Create a new user". Provided by the child class. + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + ) + self.used_in_level_2 = self.level == 2 + self.navigation_config = navigation_config + self.filter_config = None + self.protocol_name = protocol_name + self.task_description = None + self.short_description = None + self.tasks = [] + + def _setup_list(self) -> None: + """Used to create the necessary records in the list + setting up the list filter attribute before filtering it.""" + raise NotImplementedError + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self._setup_list() + config = self.fixed_config if self.fixed_config else self._get_config() + + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[AbstractServiceNowTask]: + list_url = self.filter_config["list_url"] + + navigate_to_protocol_config = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": f'Can you find the "{self.protocol_name}" Protocol in the Knowledge Base?', + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + + config = [ + # Navigate to the task start page + AllMenuTask( + instance=self.instance, + fixed_config=self.navigation_config, + is_validated=False, + used_in_level_2=True, + ), + # Filter the the list; the config it uses is set by the _setup_list method + FilterListTask( + instance=self.instance, + list_url=list_url, + fixed_config=self.filter_config, + expected_fields_path=self.filter_config["expected_fields_path"], + is_validated=False, + used_in_level_2=True, + ), + ] + self.tasks + + # To support the option of having no protocol + if self.protocol_name: + config = navigate_to_protocol_config + config + + return config diff --git a/src/browsergym/workarena/tasks/compositional/find_and_order_item.py b/src/browsergym/workarena/tasks/compositional/find_and_order_item.py new file mode 100644 index 0000000..de34e2d --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/find_and_order_item.py @@ -0,0 +1,345 @@ +from faker import Faker + +fake = Faker() + +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.send_chat_message import SendChatMessageGenericTask +from browsergym.workarena.tasks.service_catalog import ( + OrderDeveloperLaptopTask, + OrderIpadMiniTask, + OrderIpadProTask, + OrderSalesLaptopTask, + OrderStandardLaptopTask, + OrderAppleWatchTask, + OrderAppleMacBookPro15Task, + OrderDevelopmentLaptopPCTask, + OrderLoanerLaptopTask, +) + +from .base import HumanEvalTask +from .filter_and_do import FilterAndDoTask + +from ..base import AbstractServiceNowTask + +from ...api.requested_items import create_requested_item +from ...api.user import create_user +from ...api.utils import db_delete_from_table, table_api_call +from ...config import ( + # Expected columns for the different lists + EXPECTED_REQUESTED_ITEMS_COLUMNS_PATH, +) +from ...instance import SNowInstance + + +class FilterRequestedItemsAndOrderCatalogItemTask(FilterAndDoTask, HumanEvalTask): + """Generic task to filter the requested items list to find what a given user has requested and order the same thing. + Args: + fixed_request_item: str + The requested item to find and order. + task_class: AbstractServiceNowTask + The class of the task to order the item. + """ + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + fixed_request_item: str = None, + order_task_class: AbstractServiceNowTask = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "module": "Requested Items", + "application": "Self-Service", + }, + level=level, + protocol_name="", + ) + self.fixed_request_item = fixed_request_item + self.order_task_class = order_task_class + # name of the user the item will be assigned to + self.user_full_name = ( + fake.first_name() + + "-" + + fake.first_name() + + " " + + fake.last_name() + + "-" + + fake.last_name() + ) + self.short_description = f"Order same item as {self.user_full_name}" + self.task_description = f'{self.user_full_name} has recently requested an item from the service catalog. You need to order the same. Find what it is from the "Requested Items" list and order it from the service catalog. If possible, set the item\'s configuration to match the following: \n' + self.created_user_sys_id = None # sys_id of the user to assign the item to + self.requested_item_sys_id = None # sys_id of the requested item to order + self.tasks = [] + + def _setup_list(self) -> None: + self.filter_config = { + "list_url": "/now/nav/ui/classic/params/target/sc_req_item_list.do", + "expected_fields_path": EXPECTED_REQUESTED_ITEMS_COLUMNS_PATH, + "filter_columns": [ + "requested_for", + ], + "filter_kind": "AND", + "filter_operators": ["contains"], + "filter_values": [ + f"{self.user_full_name}", + ], + } + # Create a new user to assign the item to + first_name = self.user_full_name.split(" ")[0] + last_name = self.user_full_name.split(" ")[1] + _, _, self.created_user_sys_id = create_user( + self.instance, first_name=first_name, last_name=last_name, random=self.random + ) + # Create a new requested item to order + self.requested_item_sys_id, _ = create_requested_item( + self.instance, + system_name=self.fixed_request_item, + user_sys_id=self.created_user_sys_id, + ) + self.tasks.append( + # After the filter has been made and the information retrieved, navigate to the catalog + AllMenuTask( + instance=self.instance, + fixed_config={ + "module": "Service Catalog", + "application": "Self-Service", + }, + used_in_level_2=True, + is_validated=False, + ) + ) + self.tasks.append( + SendChatMessageGenericTask( + instance=self.instance, + message="a", + answer_format="a", + level=self.level, + description=f"Clear the existing filters on the page. \n", + is_validated=False, + use_description_in_l3=True, + used_in_level_2=True, + ) + ) + order_task_config = self.random.choice(self.order_task_class.all_configs()) + # task to order the item + item_order_task = self.order_task_class( + seed=self.seed, + instance=self.instance, + fixed_config=order_task_config, + used_in_level_2=True, + is_validated=True, + config_only_in_desc=True, + ) + self.tasks.append(item_order_task) + + def teardown(self) -> None: + # Delete the requested item and the user if they exist + requested_item_exists = table_api_call( + instance=self.instance, + table="sc_req_item", + params={"sysparm_query": f"sys_id={self.requested_item_sys_id}"}, + )["result"] + if requested_item_exists: + db_delete_from_table( + instance=self.instance, + table="sc_req_item", + sys_id=self.requested_item_sys_id, + ) + user_exists = table_api_call( + instance=self.instance, + table="sys_user", + params={"sysparm_query": f"sys_id={self.created_user_sys_id}"}, + )["result"] + if user_exists: + db_delete_from_table( + instance=self.instance, + table="sys_user", + sys_id=self.created_user_sys_id, + ) + + super().teardown() + + +class FilterRequestedItemsAndOrderDeveloperLaptopTask(FilterRequestedItemsAndOrderCatalogItemTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="Developer Laptop (Mac)", + level=level, + order_task_class=OrderDeveloperLaptopTask, + ) + + +class FilterRequestedItemsAndOrderIpadMiniTask(FilterRequestedItemsAndOrderCatalogItemTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="iPad mini", + level=level, + order_task_class=OrderIpadMiniTask, + ) + + +class FilterRequestedItemsAndOrderIpadProTask(FilterRequestedItemsAndOrderCatalogItemTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="iPad pro", + level=level, + order_task_class=OrderIpadProTask, + ) + + +class FilterRequestedItemsAndOrderSalesLaptopTask(FilterRequestedItemsAndOrderCatalogItemTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="Sales Laptop", + level=level, + order_task_class=OrderSalesLaptopTask, + ) + + +class FilterRequestedItemsAndOrderStandardLaptopTask(FilterRequestedItemsAndOrderCatalogItemTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="Standard Laptop", + level=level, + order_task_class=OrderStandardLaptopTask, + ) + + +class FilterRequestedItemsAndOrderAppleWatchTask(FilterRequestedItemsAndOrderCatalogItemTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="Apple Watch", + level=level, + order_task_class=OrderAppleWatchTask, + ) + + +class FilterRequestedItemsAndOrderAppleMacBookPro15Task( + FilterRequestedItemsAndOrderCatalogItemTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item='Apple MacBook Pro 15"', + level=level, + order_task_class=OrderAppleMacBookPro15Task, + ) + + +class FilterRequestedItemsAndOrderDevelopmentLaptopPCTask( + FilterRequestedItemsAndOrderCatalogItemTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="Development Laptop (PC)", + level=level, + order_task_class=OrderDevelopmentLaptopPCTask, + ) + + +class FilterRequestedItemsAndOrderLoanerLaptopTask(FilterRequestedItemsAndOrderCatalogItemTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + fixed_request_item="Notebook Computer Loaner", + level=level, + order_task_class=OrderLoanerLaptopTask, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) + and issubclass(var, FilterAndDoTask) + and var is not FilterAndDoTask + and var is not FilterRequestedItemsAndOrderCatalogItemTask +] diff --git a/src/browsergym/workarena/tasks/compositional/manage_change_request_schedule.py b/src/browsergym/workarena/tasks/compositional/manage_change_request_schedule.py new file mode 100644 index 0000000..62784e8 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/manage_change_request_schedule.py @@ -0,0 +1,1417 @@ +import re + +from datetime import datetime, timedelta +from faker import Faker +from typing import List, Tuple + +fake = Faker() + +from playwright.sync_api._generated import Page + +from browsergym.workarena.tasks.form import ( + EditChangeRequestScheduleTask, +) + +from .base import HumanEvalTask +from .filter_and_do import FilterAndDoTask + +from ..base import AbstractServiceNowTask + +from ...api.change_request import create_change_request +from ...api.utils import table_api_call, db_delete_from_table +from ...config import ( + # Expected columns for the different lists + EXPECTED_CHANGE_REQUEST_COLUMNS_PATH, +) +from ...instance import SNowInstance + + +class ManageChangeRequestScheduleTask(FilterAndDoTask): + """Task to schedule change requests. + Args: + + goal_type: str + The type of goal to set. Choices are "base", "priority", "tight", "tight priority". Used for validation + wide_schedule: bool + Whether or not the change requests should be scheduled in a 'wide' schedule. If set to True, the change requests + will have a period of 2 longer than the optimal schedule to be fitted in. Otherwise, they will have + a period of 2 days longer than the optimal schedule. + uniform_risk: bool + whether to use uniform risk for the change requests. The risk is between 2 (high) and 4 (low) and sets the + duration of the change request (high) risk=2 -> 3 days, (medium) risk=3 -> 2 days, (low) risk=4 -> 1 day + num_change_requests: int + The number of change requests to create a schedule for + pre_existing_schedule: bool + Whether to create a pre-existing schedule for the change requests. If set to True, the change requests created + will all overlap and have durations of one day. + """ + + # mapping between risk and duration + risk_to_duration = {2: 3, 3: 2, 4: 1} + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + goal_type: str = "base", + wide_schedule: bool = False, + uniform_risk: bool = True, + num_change_requests: int = 2, + pre_existing_schedule: bool = False, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "module": "All", + "application": "Change", + }, + level=level, + protocol_name="Scheduling Your Change Requests", + ) + self.goal_type = goal_type + self.wide_schedule = wide_schedule + self.uniform_risk = uniform_risk + self.num_change_requests = num_change_requests + self.pre_existing_schedule = pre_existing_schedule + self.change_request_sys_ids = [] + self.change_request_numbers = [] + self.change_request_impacts = [2] * num_change_requests # Medium priorities by default + + # start and end dates of the schedule + self.schedule_start_date = fake.date_time_this_decade( + after_now=True, before_now=False, tzinfo=None + ).replace(microsecond=0) + self.schedule_end_date = None + self.schedule_bounds_goal = None # Part of the goal to append at the end of the goal to indicate the start and end of the schedule. Used in L2 tasks + + if self.uniform_risk: + self.risks = [4] * num_change_requests + else: + self.risks = list(self.random.randint(2, 4, num_change_requests)) + + self.change_request_hashtag = "#SERIES-" + self.unique_id[:10] + if not self.pre_existing_schedule: + schedule_type = "tight schedule" if "tight" in self.goal_type else "schedule" + self.short_description = f"Scheduling Your Change Requests" + self.task_description = f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) create a {schedule_type} for your change requests for those with hashtag {self.change_request_hashtag}.' + else: + self.short_description = f"Re-scheduling Your Change Requests" + self.task_description = f'The schedule for your change requests with hashtag {self.change_request_hashtag} is currently broken. Please refer to company protocol "{self.protocol_name}" (located in the company protocols knowledge base) to fix it.' + if "tight" in self.goal_type: + self.task_description += " The change requests should be scheduled according to the tight schedule setting." + self.tasks = [] + + def setup_goal(self, page: Page) -> tuple[str, dict]: + goal, info = super().setup_goal(page=page) + + if self.level == 2: + goal += self.schedule_bounds_goal + + return goal, info + + def _setup_list(self) -> None: + self.filter_config = { + "list_url": "/now/nav/ui/classic/params/target/change_request_list.do", + "expected_fields_path": EXPECTED_CHANGE_REQUEST_COLUMNS_PATH, + "filter_columns": [ + "short_description", + ], + "filter_kind": "AND", + "filter_operators": ["contains"], + "filter_values": [ + f"{self.change_request_hashtag}", + ], + } + self.change_request_impacts.sort() # Sort the impacts to make sure the the top impact requests are scheduled first + + start_date = self.schedule_start_date + + for risk, impact in zip(self.risks, self.change_request_impacts): + if self.pre_existing_schedule: + change_request_start_date = start_date + timedelta(hours=self.random.randint(1, 4)) + change_request_end_date = change_request_start_date + timedelta(days=1) + else: + change_request_start_date = "" + change_request_end_date = "" + change_request_sys_id, change_request_number = create_change_request( + instance=self.instance, + user_sys_id=self._base_user_sysid, + risk=risk, + start_date=str(change_request_start_date), + end_date=str(change_request_end_date), + impact=impact, + hashtag=self.change_request_hashtag, + random=self.random, + ) + self.change_request_sys_ids.append(change_request_sys_id) + self.change_request_numbers.append(change_request_number) + + for i, risk in enumerate(self.risks): + skip_description = i > 0 + duration = self.risk_to_duration[risk] + end_date = start_date + timedelta(days=duration) + self.tasks.append( + EditChangeRequestScheduleTask( + instance=self.instance, + is_validated=False, + used_in_level_2=True, + record_sys_id=self.change_request_sys_ids[i], + record_number=self.change_request_numbers[i], + # Here the values will only be used by the cheat; the goal will be over-ridden to explain the task + # at a high level only; see the get_pretty_printed_description method + new_values={"start_date": str(start_date), "end_date": str(end_date)}, + level=self.level, + goal_type=self.goal_type, + skip_description=skip_description, + ) + ) + start_date = end_date + timedelta(minutes=1) + + if self.wide_schedule: + self.schedule_end_date = end_date + timedelta(weeks=2) + else: + self.schedule_end_date = end_date + timedelta(days=2) + + self.schedule_bounds_goal = f" All the change requests should be scheduled between {self.schedule_start_date} and {self.schedule_end_date}, inclusively. " + # Add the schedule bounds to the task description + self.task_description += self.schedule_bounds_goal + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + change_requests = table_api_call( + instance=self.instance, + table="change_request", + params={ + "sysparm_query": f"short_descriptionLIKE{self.change_request_hashtag}", + "sysparm_fields": "impact,start_date,end_date,risk", + }, + )["result"] + change_requests = sorted(change_requests, key=lambda x: x["start_date"]) + + # max difference is 1 day if not tight, 1 hour if tight + max_difference = 1 if self.goal_type == "tight" else 24 + + for i, change_request in enumerate(change_requests): + # Check that the change request has start/end dates + if ( + not change_request["start_date"] + or not change_request["end_date"] + or (i > 0 and not change_requests[i - 1]["end_date"]) + ): + return ( + 0, + False, + "", + {"message": "Change request start date or end date is missing."}, + ) + # Confirm that the change request has appropriate duration (within 20% of expected duration) + current_start_date = datetime.strptime( + change_request["start_date"], "%Y-%m-%d %H:%M:%S" + ) + current_end_date = datetime.strptime(change_request["end_date"], "%Y-%m-%d %H:%M:%S") + + # Check that the bounds of the schedule are respected + if ( + current_start_date < self.schedule_start_date + or current_end_date > self.schedule_end_date + ): + return ( + 0, + False, + "", + { + "message": "Change request start date or end date is outside of the target schedule." + }, + ) + + difference = current_end_date - current_start_date + # Expected duration is 3 days for high risk, 2 days for medium risk, 1 day for low risk + duration = self.risk_to_duration[int(change_request["risk"])] + expected_duration = timedelta(days=duration) + + if difference < expected_duration * 0.95 or difference > expected_duration * 1.05: + return ( + 0, + False, + "", + { + "message": "Change request duration is not within 5% of the expected duration." + }, + ) + + if i == 0: + continue + # Confirm change requests are not overlapping and respect maximum spacing (1 day if not tight, 1h if tight) + previous_end_date = datetime.strptime( + change_requests[i - 1]["end_date"], "%Y-%m-%d %H:%M:%S" + ) + difference = current_start_date - previous_end_date + if difference > timedelta(hours=max_difference) or difference < timedelta(0): + return ( + 0, + False, + "", + { + "message": "Change requests are overlapping or not respecting the maximum spacing." + }, + ) + # Confirm change requests are ordered by impact - lower number being more impactful + if change_request["impact"] > change_requests[i - 1]["impact"]: + return ( + 0, + False, + "", + {"message": "Change requests are not ordered by priority."}, + ) + + # Validate final_l3 tasks + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + + def teardown(self) -> None: + for change_request_sys_id in self.change_request_sys_ids: + record_exists = table_api_call( + instance=self.instance, + table="change_request", + params={"sysparm_query": f"sys_id={change_request_sys_id}"}, + )["result"] + if record_exists: + db_delete_from_table( + instance=self.instance, + table="change_request", + sys_id=change_request_sys_id, + ) + super().teardown() + + +class TwoChangesBasicUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesWideBasicUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesFixBasicUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesFixWideBasicUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesBasicVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesWideBasicVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesFixBasicVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesFixWideBasicVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesPriorityUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesWidePriorityUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesFixPriorityUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesFixWidePriorityUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesWidePriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesFixPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesFixWidePriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesTightUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesWideScheduleTightUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + wide_schedule=True, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesFixTightUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesFixWideScheduleTightUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesWideTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class TwoChangesFixTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class TwoChangesFixWideTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + uniform_risk=False, + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesBasicUniformRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesWideBasicUniformRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesFixBasicUniformRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesFixWideBasicUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesBasicVariedRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesWideBasicVariedRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesFixBasicVariedRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesFixWideBasicVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="base", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesPriorityUniformRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesWidePriorityUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesFixPriorityUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesFixWidePriorityUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesPriorityVariedRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesWidePriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesFixPriorityVariedRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesFixWidePriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="priority", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesTightUniformRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesWideScheduleTightUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + wide_schedule=True, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesFixTightUniformRiskChangeRequestSchedulingTask(ManageChangeRequestScheduleTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesFixWideScheduleTightUniformRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight", + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesWideTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + wide_schedule=True, + uniform_risk=False, + num_change_requests=num_change_requests, + level=level, + ) + + +class ThreeChangesFixTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + uniform_risk=False, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +class ThreeChangesFixWideTightPriorityVariedRiskChangeRequestSchedulingTask( + ManageChangeRequestScheduleTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_change_requests: int = 3, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + goal_type="tight priority", + uniform_risk=False, + wide_schedule=True, + num_change_requests=num_change_requests, + pre_existing_schedule=True, + level=level, + ) + + +local_vars = locals().copy() + + +SMALL_BASE_SCHEDULING_TASKS = [ + TwoChangesBasicUniformRiskChangeRequestSchedulingTask, + TwoChangesWideBasicUniformRiskChangeRequestSchedulingTask, + TwoChangesFixBasicUniformRiskChangeRequestSchedulingTask, + TwoChangesFixWideBasicUniformRiskChangeRequestSchedulingTask, + TwoChangesBasicVariedRiskChangeRequestSchedulingTask, + TwoChangesWideBasicVariedRiskChangeRequestSchedulingTask, + TwoChangesFixBasicVariedRiskChangeRequestSchedulingTask, + TwoChangesFixWideBasicVariedRiskChangeRequestSchedulingTask, +] +SMALL_TIGHT_SCHEDULING_TASKS = [ + TwoChangesPriorityUniformRiskChangeRequestSchedulingTask, + TwoChangesWidePriorityUniformRiskChangeRequestSchedulingTask, + TwoChangesFixPriorityUniformRiskChangeRequestSchedulingTask, + TwoChangesFixWidePriorityUniformRiskChangeRequestSchedulingTask, + TwoChangesPriorityVariedRiskChangeRequestSchedulingTask, + TwoChangesWidePriorityVariedRiskChangeRequestSchedulingTask, + TwoChangesFixPriorityVariedRiskChangeRequestSchedulingTask, + TwoChangesFixWidePriorityVariedRiskChangeRequestSchedulingTask, + TwoChangesTightUniformRiskChangeRequestSchedulingTask, + TwoChangesWideScheduleTightUniformRiskChangeRequestSchedulingTask, + TwoChangesFixTightUniformRiskChangeRequestSchedulingTask, + TwoChangesFixWideScheduleTightUniformRiskChangeRequestSchedulingTask, + TwoChangesTightPriorityVariedRiskChangeRequestSchedulingTask, + TwoChangesWideTightPriorityVariedRiskChangeRequestSchedulingTask, + TwoChangesFixTightPriorityVariedRiskChangeRequestSchedulingTask, + TwoChangesFixWideTightPriorityVariedRiskChangeRequestSchedulingTask, +] + +LARGE_BASE_SCHEDULING_TASKS = [ + ThreeChangesBasicUniformRiskChangeRequestSchedulingTask, + ThreeChangesWideBasicUniformRiskChangeRequestSchedulingTask, + ThreeChangesFixBasicUniformRiskChangeRequestSchedulingTask, + ThreeChangesFixWideBasicUniformRiskChangeRequestSchedulingTask, + ThreeChangesBasicVariedRiskChangeRequestSchedulingTask, + ThreeChangesWideBasicVariedRiskChangeRequestSchedulingTask, + ThreeChangesFixBasicVariedRiskChangeRequestSchedulingTask, + ThreeChangesFixWideBasicVariedRiskChangeRequestSchedulingTask, +] + +LARGE_TIGHT_SCHEDULING_TASKS = [ + ThreeChangesPriorityUniformRiskChangeRequestSchedulingTask, + ThreeChangesWidePriorityUniformRiskChangeRequestSchedulingTask, + ThreeChangesFixPriorityUniformRiskChangeRequestSchedulingTask, + ThreeChangesFixWidePriorityUniformRiskChangeRequestSchedulingTask, + ThreeChangesPriorityVariedRiskChangeRequestSchedulingTask, + ThreeChangesWidePriorityVariedRiskChangeRequestSchedulingTask, + ThreeChangesFixPriorityVariedRiskChangeRequestSchedulingTask, + ThreeChangesFixWidePriorityVariedRiskChangeRequestSchedulingTask, + ThreeChangesTightUniformRiskChangeRequestSchedulingTask, + ThreeChangesWideScheduleTightUniformRiskChangeRequestSchedulingTask, + ThreeChangesFixTightUniformRiskChangeRequestSchedulingTask, + ThreeChangesFixWideScheduleTightUniformRiskChangeRequestSchedulingTask, + ThreeChangesTightPriorityVariedRiskChangeRequestSchedulingTask, + ThreeChangesWideTightPriorityVariedRiskChangeRequestSchedulingTask, + ThreeChangesFixTightPriorityVariedRiskChangeRequestSchedulingTask, + ThreeChangesFixWideTightPriorityVariedRiskChangeRequestSchedulingTask, +] + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) + and issubclass(var, FilterAndDoTask) + and var is not FilterAndDoTask + and var is not ManageChangeRequestScheduleTask +] diff --git a/src/browsergym/workarena/tasks/compositional/mark_duplicate_problems.py b/src/browsergym/workarena/tasks/compositional/mark_duplicate_problems.py new file mode 100644 index 0000000..70ea856 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/mark_duplicate_problems.py @@ -0,0 +1,499 @@ +from faker import Faker + +fake = Faker() + +from playwright.sync_api._generated import Page + +from .base import HumanEvalTask +from .filter_and_do import FilterAndDoTask + +from ..mark_duplicate_problem import SetProblemAsDuplicateTask +from ..base import AbstractServiceNowTask + +from ...api.problem import create_problem +from ...api.utils import db_delete_from_table, table_api_call +from ...config import ( + # Expected columns for the different lists + EXPECTED_PROBLEM_COLUMNS_PATH, +) +from ...instance import SNowInstance + + +class FilterProblemsAndMarkDuplicatesTask(FilterAndDoTask): + """Basic task to filter problems with a specific hashtag and mark them as duplicates.""" + + def __init__( + self, + seed: int, + extra_problems: int, + navigation_config: dict, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config=navigation_config, + level=level, + protocol_name="Problem List Cleanup", + ) + self.extra_problems = extra_problems # Number of non-duplicates to create; total problems will be extra_problems + 2 + self.problem_priorities = [2, 2] + [ + 1 + ] * extra_problems # The first two problems will be duplicates with the same priority; the other ones will get top priority by default + self.problem_sys_ids = [] + self.duplicate_problems = [] + + self.problem_hashtag = "#SERIES-" + self.unique_id[:10] + self.short_description = f"Clean-up your duplicate problems" + self.task_description = f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) clean-up your problem list (problems assigned to you) by marking duplicate problems among those with hashtag {self.problem_hashtag}.' + + def _setup_list(self) -> None: + duplicated_short_descripton = f"{fake.sentence(4)}" + + for i, priority in enumerate(self.problem_priorities): + # The first two problems will have the same short description; the other ones will get random ones + if i < 2: + short_description = duplicated_short_descripton + else: + short_description = None + + problem_sys_id, problem_number = create_problem( + instance=self.instance, + problem_hashtag=self.problem_hashtag, + priority=priority, + user_sys_id=self._base_user_sysid, + short_description=short_description, + return_number=True, + ) + self.problem_sys_ids.append(problem_sys_id) + + if i < 2: + self.duplicate_problems.append({"number": problem_number, "sys_id": problem_sys_id}) + + self.filter_config = { + "list_url": "/now/nav/ui/classic/params/target/problem_list.do", + "expected_fields_path": EXPECTED_PROBLEM_COLUMNS_PATH, + "filter_columns": [ + "short_description", + ], + "filter_kind": "AND", + "filter_operators": ["contains"], + "filter_values": [ + f"{self.problem_hashtag}", + ], + } + # the 'tasks' attribute needs to be defined by children classes + + def teardown(self) -> None: + for problem_sys_id in self.problem_sys_ids: + record_exists = table_api_call( + instance=self.instance, + table="problem", + params={"sysparm_query": f"sys_id={problem_sys_id}"}, + )["result"] + if record_exists: + db_delete_from_table( + instance=self.instance, + table="problem", + sys_id=problem_sys_id, + ) + super().teardown() + + +class BasicFilterProblemsAndMarkDuplicatesSmallTask( + FilterProblemsAndMarkDuplicatesTask, HumanEvalTask +): + """Basic task to filter problems with a specific hashtag and mark them as duplicates. This""" + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "Assigned to me", + "application": "Problem", + }, + level=level, + ) + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + is_validated=True, + used_in_level_2=True, + goal_version="base", + level=self.level, + ), + ] + + +class BasicFilterProblemsAndMarkDuplicatesMediumTask(FilterProblemsAndMarkDuplicatesTask): + """Basic task to filter problems with a specific hashtag and mark them as duplicates. This""" + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 4, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "Assigned to me", + "application": "Problem", + }, + level=level, + ) + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + is_validated=True, + used_in_level_2=True, + goal_version="base", + level=self.level, + ), + ] + + +class BasicFilterProblemsAndMarkDuplicatesLargeTask(FilterProblemsAndMarkDuplicatesTask): + """Basic task to filter problems with a specific hashtag and mark them as duplicates. This""" + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 6, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "Assigned to me", + "application": "Problem", + }, + level=level, + ) + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + is_validated=True, + used_in_level_2=True, + goal_version="base", + level=self.level, + ), + ] + + +class PriorityFilterProblemsAndMarkDuplicatesSmallTask( + FilterProblemsAndMarkDuplicatesTask, HumanEvalTask +): + """Task to filter problems with a specific hashtag and mark the least priority one as duplicate of the first.""" + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "All", + "application": "Problem", + }, + level=level, + ) + self.problem_priorities = [1, 2] + [1] * extra_problems + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + respect_problem_ordering=True, + is_validated=True, + used_in_level_2=True, + goal_version="priority", + level=self.level, + ), + ] + + +class PriorityFilterProblemsAndMarkDuplicatesMediumTask(FilterProblemsAndMarkDuplicatesTask): + """Task to filter problems with a specific hashtag and mark the least priority one as duplicate of the first.""" + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 4, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "All", + "application": "Problem", + }, + level=level, + ) + self.problem_priorities = [1, 2] + [1] * extra_problems + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + respect_problem_ordering=True, + is_validated=True, + used_in_level_2=True, + goal_version="priority", + level=self.level, + ), + ] + + +class PriorityFilterProblemsAndMarkDuplicatesLargeTask(FilterProblemsAndMarkDuplicatesTask): + """Task to filter problems with a specific hashtag and mark the least priority one as duplicate of the first.""" + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 6, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "All", + "application": "Problem", + }, + level=level, + ) + self.problem_priorities = [1, 2] + [1] * extra_problems + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + respect_problem_ordering=True, + is_validated=True, + used_in_level_2=True, + goal_version="priority", + level=self.level, + ), + ] + + +class HighPriorityFilterProblemsAndMarkDuplicatesSmallTask( + FilterProblemsAndMarkDuplicatesTask, HumanEvalTask +): + """Task to filter problems with a specific hashtag and mark high priority items as duplicates. As + a top priority item is marked as duplicate, we have to add a comment to it. + """ + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 2, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "All", + "application": "Problem", + }, + level=level, + ) + self.problem_priorities = [1, 1] + [1] * extra_problems + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + respect_problem_ordering=False, + is_validated=True, + used_in_level_2=True, + goal_version="high priority", + level=self.level, + ), + ] + + +class HighPriorityFilterProblemsAndMarkDuplicatesMediumTask(FilterProblemsAndMarkDuplicatesTask): + """Task to filter problems with a specific hashtag and mark high priority items as duplicates. As + a top priority item is marked as duplicate, we have to add a comment to it. + """ + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 4, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "All", + "application": "Problem", + }, + level=level, + ) + self.problem_priorities = [1, 1] + [1] * extra_problems + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + respect_problem_ordering=False, + is_validated=True, + used_in_level_2=True, + goal_version="high priority", + level=self.level, + ), + ] + + +class HighPriorityFilterProblemsAndMarkDuplicatesLargeTask(FilterProblemsAndMarkDuplicatesTask): + """Task to filter problems with a specific hashtag and mark high priority items as duplicates. As + a top priority item is marked as duplicate, we have to add a comment to it. + """ + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + extra_problems: int = 6, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + extra_problems=extra_problems, + fixed_config=fixed_config, + navigation_config={ + "module": "All", + "application": "Problem", + }, + level=level, + ) + self.problem_priorities = [1, 1] + [1] * extra_problems + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SetProblemAsDuplicateTask( + instance=self.instance, + fixed_config={ + "target_problem": self.duplicate_problems[1], + "source_problem": self.duplicate_problems[0], + }, + respect_problem_ordering=False, + is_validated=True, + used_in_level_2=True, + goal_version="high priority", + level=self.level, + ), + ] + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) + and issubclass(var, FilterAndDoTask) + and var is not FilterAndDoTask + and var is not FilterProblemsAndMarkDuplicatesTask +] diff --git a/src/browsergym/workarena/tasks/compositional/maximize_investment_return.py b/src/browsergym/workarena/tasks/compositional/maximize_investment_return.py new file mode 100644 index 0000000..605eba4 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/maximize_investment_return.py @@ -0,0 +1,1763 @@ +import re + +from faker import Faker +from typing import List, Tuple + +fake = Faker() + +from playwright.sync_api._generated import Page + +from browsergym.workarena.tasks.send_chat_message import SendChatMessageForBudgetAllocationTask + +from .base import HumanEvalTask +from .delete_record import DeleteExpenseLineKnapsack +from .filter_and_do import FilterAndDoTask +from .utils.knapsack import KnapsackInstanceGenarator + +from ..base import AbstractServiceNowTask + +from ...api.expense_line import create_expense_line +from ...api.utils import table_api_call, db_delete_from_table +from ...config import ( + # Expected columns for the different lists + EXPECTED_EXPENSE_LINE_COLUMNS_PATH, +) +from ...instance import SNowInstance + + +class FilterExpensesAndAllocateInvestmentsTask(FilterAndDoTask): + """Task to filter expenses and allocate investments. + Args: + num_expenses: list[int] + The range to choose the number of expenses from + budget: int + The budget to allocate to the expenses + mode: str + Mode of generation. Choice of "random", "trivial", "single_item", "single_item_uniform", "n_items" + - random: Randomly generate the instance and return it; guaranteed to have a unique optimal solution + - trivial: Generate a trivial instance with all items fitting in the knapsack; return the instance + - single_item: Generate an instance where the optimal solution has only one item + - n_items: Generate an instance with all items having uniform weight and value; n items fitting in the knapsack + - single_item_uniform: Generate an instance with all items having uniform weight and value; optimal solution has only one item and it can be any + answer_format: str + The type of answer to generate. Choice of total_return_only, total_return_and_investments, investments_only, cleanup, cleanup_and_return + num_items_uniform: int + The number of items to generate in the "n_items" mode + """ + + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + num_expenses: list[int] = [3, 4], + budget: int = 150000, + mode: str = "random", + num_items_uniform: int = None, + answer_format: str = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "module": "Expense Lines", + "application": "Cost", + }, + level=level, + protocol_name="Maximizing total investment return", + ) + self.num_expenses = self.random.randint(num_expenses[0], num_expenses[1] + 1) + # In these settings, we need to vary the budget + if mode in ["single_item_uniform", "n_items"]: + min_budget = budget / 5 + max_budget = budget * 5 + self.budget = self.random.randint(min_budget, max_budget) + else: + self.budget = budget + self.mode = mode + self.answer_format = answer_format + self.num_items_uniform = 1 if mode == "single_item_uniform" else num_items_uniform + + self.expense_hashtag = "#" + self.unique_id[:10] + self.short_description = f"Allocate investments to maximize returns" + self.expense_line_sys_ids = [] + self.expense_line_numbers = [] + self.correct_investments = ( + [] + ) # List of correct investments to check for in the chat messages + self.incorrect_investments = ( + [] + ) # List of incorrect investments to check for in the chat messages + self.potential_investments = None # List of tuples (cost, return) of potential investments + self.max_return = None # Maximum return possible with optimal solution + self.alternative_max_return_formats = ( + [] + ) # List of alternative formats for the maximum return to check for in the chat messages + self.selected_investment_indices = ( + None # Indices of the selected investments in the optimal solution + ) + # flag to check if the investments are correctly selected and total return is correct + self.investments_correctly_selected = False + self.total_return_correct = False + + def _setup_list(self) -> None: + self.filter_config = { + "list_url": "/now/nav/ui/classic/params/target/fm_expense_line_list.do", + "expected_fields_path": EXPECTED_EXPENSE_LINE_COLUMNS_PATH, + "filter_columns": [ + "short_description", + ], + "filter_kind": "AND", + "filter_operators": ["contains"], + "filter_values": [ + f"{self.expense_hashtag}", + ], + } + knapsack = KnapsackInstanceGenarator( + random=self.random, + num_items=self.num_expenses, + max_capacity=self.budget, + mode=self.mode, + num_items_in_solution=self.num_items_uniform, + ) + # investments is a list of tuples, where each tuple is (cost, return) + self.potential_investments, self.max_return, self.selected_investment_indices = ( + knapsack.get_instance() + ) + # Accepted answer formats for the maximum return + self.alternative_max_return_formats = [ + str(self.max_return), # No comma + "{:,}".format(self.max_return), # Comma as thousand separator + "{:,}".format(self.max_return).replace( + ",", ", " + ), # Comma as thousand separator with space after + "{:,}".format(self.max_return).replace(",", " "), # Space as thousand separator + ] + + for i, investment in enumerate(self.potential_investments): + expense_number = f"EXP-{i}{self.unique_id[:10]}" + # Include the return inside the short description + short_description = f"Build {fake.sentence(2)} - Return: {investment[1]}$ " + expense_sys_id, expense_number = create_expense_line( + instance=self.instance, + amount=investment[0], + number=expense_number, + date=str(fake.date_this_year(before_today=True, after_today=False)), + short_description=short_description, + expense_hashtag=self.expense_hashtag, + user_sys_id=self._base_user_sysid, + ) + self.expense_line_sys_ids.append(expense_sys_id) + self.expense_line_numbers.append(expense_number) + + # In this setting there is only one valid answer + if self.mode in ["random", "trivial", "single_item"]: + for i, investment in enumerate(self.potential_investments): + if i in self.selected_investment_indices: + self.correct_investments.append(self.expense_line_numbers[i]) + else: + self.incorrect_investments.append(self.expense_line_numbers[i]) + # In this setting, many answers are possible, it's only a matter of respecting the number of items in the solution + # We store values here just so the cheat function can work uniformly + elif self.mode in ["n_items", "single_item_uniform"]: + for i, investment in enumerate(self.potential_investments): + if i < self.num_items_uniform: + self.correct_investments.append(self.expense_line_numbers[i]) + else: + self.incorrect_investments.append(self.expense_line_numbers[i]) + + def validate(self, page: Page, chat_messages: List[str]) -> Tuple[float, bool, str, dict]: + super().validate(page, chat_messages) + + def check_total_return( + self, page: Page, chat_messages: List[str] + ) -> Tuple[float, bool, str, dict]: + """Simple check that validates that the total return is correct.""" + if self.total_return_correct: + return ( + 1, + True, + "That is correct, thank you!", + {"message": "Correct total return."}, + ) + + if chat_messages and chat_messages[-1]["role"] == "assistant": + answer = chat_messages[-1]["message"] + else: + return ( + 0, + False, + "", + {"message": "The assistant did not provide an answer."}, + ) + for format in self.alternative_max_return_formats: + if format in answer: + self.total_return_correct = True + return ( + 1, + True, + "That is correct, thank you!", + {"message": "Correct answer."}, + ) + + return ( + 0, + False, + "", + {"message": "Incorrect answer."}, + ) + + def check_correct_investments_sent_in_chat( + self, page: Page, chat_messages: List[str] + ) -> Tuple[float, bool, str, dict]: + """Check that the correct investments have been selected and their numbers have been sent in the chat""" + if not self.investments_correctly_selected: + if chat_messages and chat_messages[-1]["role"] == "assistant": + answer = chat_messages[-1]["message"] + else: + return ( + 0, + False, + "", + {"message": "The assistant did not provide an answer."}, + ) + + # In these settings, there is only one valid answer + if self.mode in ["random", "trivial", "single_item"]: + # Check that the correct investments have been selected + for investment in self.correct_investments: + if investment not in answer: + return ( + 0, + False, + "", + {"message": "Investment missing from selected list."}, + ) + # Check that the incorrect investments have not been selected + for investment in self.incorrect_investments: + if investment in answer: + return ( + 0, + False, + "", + {"message": "Incorrect investment selected."}, + ) + # In those settings, many answers are possible, it's only a matter of respecting the number of items in the solution + elif self.mode in ["n_items", "single_item_uniform"]: + # Extract the expense line numbers from the answer + pattern = r"EXP-\w+-\w+" + matches = re.findall(pattern, answer) + if len(matches) != self.num_items_uniform: + return ( + 0, + False, + "", + {"message": "Incorrect number of investments selected."}, + ) + self.correct_investments_selected = True + + return ( + 1, + True, + "That is correct, thank you!", + {"message": "Correct investments selected."}, + ) + + def check_only_right_investment_kept( + self, page: Page, chat_messages: List[str] + ) -> Tuple[float, bool, str, dict]: + """Checks that only the expected investments were kept; i.e. the others were deleted""" + for i, investment_sys_id in enumerate(self.expense_line_sys_ids): + record_expected = i in self.selected_investment_indices + record_exists = table_api_call( + instance=self.instance, + table="fm_expense_line", + params={"sysparm_query": f"sys_id={investment_sys_id}"}, + )["result"] + # Missing investment that should be kept + if record_expected and not record_exists: + return ( + 0, + True, + "", + {"message": "Expected investment has been deleted."}, + ) + # Unexpected investment that should be deleted + if not record_expected and record_exists: + return ( + 0, + False, + "", + {"message": "Unexpected investment is present."}, + ) + + return ( + 1, + True, + "That is correct, thank you!", + {"message": "Correct investments kept."}, + ) + + def teardown(self) -> None: + for expense_sys_id in self.expense_line_sys_ids: + record_exists = table_api_call( + instance=self.instance, + table="fm_expense_line", + params={"sysparm_query": f"sys_id={expense_sys_id}"}, + )["result"] + if record_exists: + db_delete_from_table( + instance=self.instance, + table="fm_expense_line", + sys_id=expense_sys_id, + ) + super().teardown() + + +class FilterExpensesAndFindTotalReturnTask(FilterExpensesAndAllocateInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + num_expenses: list[int] = [3, 4], + budget: int = 150000, + mode: str = "random", + answer_format: str = "total_return_only", + num_items_uniform: int = 1, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=num_expenses, + budget=budget, + mode=mode, + num_items_uniform=num_items_uniform, + answer_format=answer_format, + level=level, + ) + self.task_description = f'Follow protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) to allocate investments to the expenses with short description containing {self.expense_hashtag} to maximize returns while fitting inside the budget of {self.budget}$. Give total return of selected investments only. ' + + def _setup_list(self) -> None: + super()._setup_list() + self.tasks = [ + SendChatMessageForBudgetAllocationTask( + instance=self.instance, + message=f"The total value of the investments is {self.max_return}$", + used_in_level_2=True, + is_validated=False, + budget=self.budget, + answer_format=self.answer_format, + level=self.level, + ) + ] + + def validate(self, page: Page, chat_messages: List[str]) -> Tuple[float, bool, str, dict]: + reward, done, message, info = self.check_total_return(page, chat_messages) + if reward == 1 and done: + return FilterAndDoTask.validate(self, page, chat_messages) + else: + return reward, done, message, info + + +class FilterRandomExpensesAndFindTotalReturnSmallTask( + FilterExpensesAndFindTotalReturnTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesAndFindTotalReturnMediumTask(FilterExpensesAndFindTotalReturnTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesAndFindTotalReturnLargeTask(FilterExpensesAndFindTotalReturnTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="random", + level=level, + ) + + +class FilterTrivialExpensesAndFindTotalReturnSmallTask( + FilterExpensesAndFindTotalReturnTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterTrivialExpensesAndFindTotalReturnMediumTask(FilterExpensesAndFindTotalReturnTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterTrivialExpensesAndFindTotalReturnLargeTask(FilterExpensesAndFindTotalReturnTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterSingleItemExpensesAndFindTotalReturnSmallTask( + FilterExpensesAndFindTotalReturnTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesAndFindTotalReturnMediumTask(FilterExpensesAndFindTotalReturnTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesAndFindTotalReturnLargeTask(FilterExpensesAndFindTotalReturnTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndFindTotalReturnSmallTask( + FilterExpensesAndFindTotalReturnTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndFindTotalReturnMediumTask( + FilterExpensesAndFindTotalReturnTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndFindTotalReturnLargeTask( + FilterExpensesAndFindTotalReturnTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterTwoItemsUniformExpensesAndFindTotalReturnSmallTask( + FilterExpensesAndFindTotalReturnTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=2, + ) + + +class FilterThreeItemsUniformExpensesAndFindTotalReturnMediumTask( + FilterExpensesAndFindTotalReturnTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +class FilterThreeItemsUniformExpensesAndFindTotalReturnLargeTask( + FilterExpensesAndFindTotalReturnTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +class FilterExpensesAndSelectInvestmentsTask(FilterExpensesAndAllocateInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + num_expenses: list[int] = [3, 4], + budget: int = 150000, + mode: str = "random", + num_items_uniform: int = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=num_expenses, + budget=budget, + mode=mode, + level=level, + answer_format="investments_only", + num_items_uniform=num_items_uniform, + ) + self.task_description = f'Follow protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) to allocate investments to the expenses with short description containing {self.expense_hashtag} to maximize returns while fitting inside the budget of {self.budget}$. Give selected investments only. ' + + def _setup_list(self) -> None: + super()._setup_list() + message = f"The correct investments to select are: {', '.join(self.correct_investments)}" + self.tasks.append( + SendChatMessageForBudgetAllocationTask( + instance=self.instance, + message=message, + used_in_level_2=True, + is_validated=False, + budget=self.budget, + answer_format=self.answer_format, + level=self.level, + ) + ) + + def validate(self, page: Page, chat_messages: List[str]) -> Tuple[float, bool, str, dict]: + reward, done, message, info = self.check_correct_investments_sent_in_chat( + page, chat_messages + ) + if reward == 1 and done: + return FilterAndDoTask.validate(self, page, chat_messages) + else: + return reward, done, message, info + + +class FilterRandomExpensesAndSelectInvestmentsSmallTask( + FilterExpensesAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesAndSelectInvestmentsMediumTask(FilterExpensesAndSelectInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesAndSelectInvestmentsLargeTask(FilterExpensesAndSelectInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="random", + level=level, + ) + + +class FilterTrivialExpensesAndSelectInvestmentsSmallTask( + FilterExpensesAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterTrivialExpensesAndSelectInvestmentsMediumTask(FilterExpensesAndSelectInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterTrivialExpensesAndSelectInvestmentsLargeTask(FilterExpensesAndSelectInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterSingleItemExpensesAndSelectInvestmentsSmallTask( + FilterExpensesAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesAndSelectInvestmentsMediumTask( + FilterExpensesAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesAndSelectInvestmentsLargeTask(FilterExpensesAndSelectInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndSelectInvestmentsSmallTask( + FilterExpensesAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndSelectInvestmentsMediumTask( + FilterExpensesAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndSelectInvestmentsLargeTask( + FilterExpensesAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterTwoItemsUniformExpensesAndSelectInvestmentsSmallTask( + FilterExpensesAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=2, + ) + + +class FilterThreeItemsUniformExpensesAndSelectInvestmentsMediumTask( + FilterExpensesAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +class FilterThreeItemsUniformExpensesAndSelectInvestmentsLargeTask( + FilterExpensesAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +class FilterExpensesFindTotalReturnAndSelectInvestmentsTask(FilterExpensesAndFindTotalReturnTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_expenses: list[int] = [3, 4], + budget: int = 150000, + mode="random", + num_items_uniform: int = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=num_expenses, + budget=budget, + mode=mode, + num_items_uniform=num_items_uniform, + answer_format="total_return_and_investments", + level=level, + ) + self.task_description = f'Follow protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) to allocate investments to the expenses with short description containing {self.expense_hashtag} to maximize returns while fitting inside the budget of {self.budget}$. Give selected investments and total return. ' + + def _setup_list(self) -> None: + super()._setup_list() + message = f"The correct investments to select are: {', '.join(self.correct_investments)} and their total return is {self.max_return}$" + self.tasks = [ + SendChatMessageForBudgetAllocationTask( + instance=self.instance, + message=message, + used_in_level_2=True, + is_validated=False, + budget=self.budget, + answer_format=self.answer_format, + level=self.level, + ) + ] + + def validate(self, page: Page, chat_messages: List[str]) -> Tuple[float, bool, str, dict]: + reward, done, message, info = self.check_correct_investments_sent_in_chat( + page, chat_messages + ) + if not (reward == 1 and done): + return reward, done, message, info + + reward, done, message, info = self.check_total_return(page, chat_messages) + if not (reward == 1 and done): + return reward, done, message, info + + return FilterAndDoTask.validate(self, page, chat_messages) + + +class FilterRandomExpensesFindTotalReturnAndSelectInvestmentsSmallTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesFindTotalReturnAndSelectInvestmentsMediumTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesFindTotalReturnAndSelectInvestmentsLargeTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="random", + level=level, + ) + + +class FilterTrivialExpensesFindTotalReturnAndSelectInvestmentsSmallTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterTrivialExpensesFindTotalReturnAndSelectInvestmentsMediumTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterTrivialExpensesFindTotalReturnAndSelectInvestmentsLargeTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="trivial", + level=level, + ) + + +class FilterSingleItemExpensesFindTotalReturnAndSelectInvestmentsSmallTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesFindTotalReturnAndSelectInvestmentsMediumTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesFindTotalReturnAndSelectInvestmentsLargeTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemUniformExpensesFindTotalReturnAndSelectInvestmentsSmallTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesFindTotalReturnAndSelectInvestmentsMediumTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesFindTotalReturnAndSelectInvestmentsLargeTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterTwoItemsUniformExpensesFindTotalReturnAndSelectInvestmentsSmallTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=2, + ) + + +class FilterThreeItemsUniformExpensesFindTotalReturnAndSelectInvestmentsMediumTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +class FilterThreeItemsUniformExpensesFindTotalReturnAndSelectInvestmentsLargeTask( + FilterExpensesFindTotalReturnAndSelectInvestmentsTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +class FilterExpenseLinesAndDeleteWrongInvestments(FilterExpensesAndAllocateInvestmentsTask): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + num_expenses: List[int] = [3, 4], + budget: int = 150000, + mode: str = "random", + num_items_uniform: int = None, + level: int = 2, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + num_expenses=num_expenses, + budget=budget, + mode=mode, + answer_format="cleanup", + num_items_uniform=num_items_uniform, + level=level, + ) + self.task_description = f'Follow protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) to allocate investments to the expenses with short description containing {self.expense_hashtag} to maximize returns while fitting inside the budget of {self.budget}$. Delete the investments that were not selected. ' + + def _setup_list(self) -> None: + super()._setup_list() + # in modes "n_items", "single_item_uniform", this yields one of many valid solutions + for i, expense_line_number in enumerate(self.incorrect_investments): + skip_description = i > 0 + expense_line_sys_id = self.expense_line_sys_ids[i] + self.tasks.append( + DeleteExpenseLineKnapsack( + instance=self.instance, + record_number=expense_line_number, + record_sys_id=expense_line_sys_id, + fixed_config={ + "field_name": "number", + "field_value": f"{expense_line_number}", + }, + used_in_level_2=True, + is_validated=False, + budget=self.budget, + answer_format=self.answer_format, + level=self.level, + skip_description=skip_description, + ) + ) + + def validate(self, page: Page, chat_messages: List[str]) -> Tuple[float, bool, str, dict]: + expenses = table_api_call( + instance=self.instance, + table="fm_expense_line", + params={ + "sysparm_query": f"short_descriptionLIKE{self.expense_hashtag}", + "sysparm_fields": "number,amount,sys_id", + }, + )["result"] + + if self.mode in ["random", "trivial", "single_item"]: + # Check that the correct investments have been selected + for investment in self.correct_investments: + if investment not in [expense["number"] for expense in expenses]: + return ( + 0, + False, + "", + {"message": "Investment missing from selected list."}, + ) + # Check that the incorrect investments have not been selected + for investment in self.incorrect_investments: + if investment in [expense["number"] for expense in expenses]: + return ( + 0, + False, + "", + {"message": "Incorrect investment selected."}, + ) + # In those settings, many answers are possible, it's only a matter of respecting the number of items in the solution + elif self.mode in ["n_items", "single_item_uniform"]: + if len(expenses) != self.num_items_uniform: + return ( + 0, + False, + "", + {"message": "Incorrect number of investments selected."}, + ) + reward, done, message, info = FilterAndDoTask.validate(self, page, chat_messages) + + return reward, done, message, info + + +class FilterRandomExpensesAndDeleteWrongInvestmentsSmallTask( + FilterExpenseLinesAndDeleteWrongInvestments, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesAndDeleteWrongInvestmentsMediumTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="random", + level=level, + ) + + +class FilterRandomExpensesAndDeleteWrongInvestmentsLargeTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="random", + level=level, + ) + + +class FilterSingleItemExpensesAndDeleteWrongInvestmentsSmallTask( + FilterExpenseLinesAndDeleteWrongInvestments, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesAndDeleteWrongInvestmentsMediumTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemExpensesAndDeleteWrongInvestmentsLargeTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndDeleteWrongInvestmentsSmallTask( + FilterExpenseLinesAndDeleteWrongInvestments, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndDeleteWrongInvestmentsMediumTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterSingleItemUniformExpensesAndDeleteWrongInvestmentsLargeTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="single_item_uniform", + level=level, + ) + + +class FilterTwoItemsUniformExpensesAndDeleteWrongInvestmentsSmallTask( + FilterExpenseLinesAndDeleteWrongInvestments, HumanEvalTask +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[3, 5], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=2, + ) + + +class FilterThreeItemsUniformExpensesAndDeleteWrongInvestmentsMediumTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[6, 8], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +class FilterThreeItemsUniformExpensesAndDeleteWrongInvestmentsLargeTask( + FilterExpenseLinesAndDeleteWrongInvestments +): + def __init__( + self, + seed: int, + instance: SNowInstance = None, + fixed_config: List[AbstractServiceNowTask] = None, + level: int = 2, + ): + super().__init__( + seed, + instance, + fixed_config, + num_expenses=[9, 12], + budget=150000, + mode="n_items", + level=level, + num_items_uniform=3, + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) + and issubclass(var, FilterAndDoTask) + and var is not FilterAndDoTask + and var is not FilterExpensesAndAllocateInvestmentsTask + and var is not FilterExpensesAndFindTotalReturnTask + and var is not FilterExpenseLinesAndDeleteWrongInvestments + and var is not FilterExpensesFindTotalReturnAndSelectInvestmentsTask + and var is not FilterExpensesAndSelectInvestmentsTask +] diff --git a/src/browsergym/workarena/tasks/compositional/navigate_and_do.py b/src/browsergym/workarena/tasks/compositional/navigate_and_do.py new file mode 100644 index 0000000..4cf3ee2 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/navigate_and_do.py @@ -0,0 +1,1151 @@ +import json + +from faker import Faker + +fake = Faker() + +from playwright.sync_api._generated import Page + +from browsergym.workarena.tasks.form import ( + CreateChangeRequestTask, + CreateHardwareAssetTask, + CreateIncidentTask, + CreateProblemTask, + CreateUserTask, +) +from browsergym.workarena.tasks.list import ( + FilterAssetListTask, + FilterChangeRequestListTask, + FilterHardwareListTask, + FilterIncidentListTask, + FilterServiceCatalogItemListTask, + FilterUserListTask, + SortAssetListTask, + SortChangeRequestListTask, + SortHardwareListTask, + SortIncidentListTask, + SortServiceCatalogItemListTask, + SortUserListTask, +) +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.service_catalog import ( + OrderDeveloperLaptopTask, + OrderIpadMiniTask, + OrderIpadProTask, + OrderSalesLaptopTask, + OrderStandardLaptopTask, + OrderAppleWatchTask, + OrderAppleMacBookPro15Task, + OrderDevelopmentLaptopPCTask, + OrderLoanerLaptopTask, +) + +from .base import CompositionalTask, HumanEvalTask + +from ..base import AbstractServiceNowTask + +from ...instance import SNowInstance + + +class NavigateAndDoTask(CompositionalTask, HumanEvalTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + navigation_config: dict = None, + level: int = 2, + task: AbstractServiceNowTask = None, + ) -> None: + """ + Generic task to navigate to a specific page and perform a task. + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of tuples, each containing a subtask + navigation_config: dict + Configuration to use for the navigation task. Contains the application and the module; the URL is not necessary as the + nav step is not validated. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + task: AbstractServiceNowTask + The task to perform after navigating to the page. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. Provided by the child class. + short_description: str + A short description of the task to be completed. "Create a new user". Provided by the child class. + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + ) + self.used_in_level_2 = self.level == 2 + self.task = task + self.task_description = None + self.short_description = None + # Get the navigation configuration; there is only one configuration for each application and module combo + self.navigation_config = navigation_config + + def setup_goal(self, page: Page) -> tuple[str, dict]: + config = self.fixed_config if self.fixed_config else self._get_config() + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[AbstractServiceNowTask]: + + config = [ + # Navigate to the task start page + AllMenuTask( + instance=self.instance, + fixed_config=self.navigation_config, + is_validated=False, + used_in_level_2=True, + has_description=True, + ), + self.task, + ] + + return config + + +class NavigateAndCreateUserTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the user list page and create a new user. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new user" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + task=CreateUserTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Create a new user with the required information. \n" + self.short_description = "Create a new user" + + +class NavigateAndCreateIncidentTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the incident list page and create a new incident. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new incident" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + task=CreateIncidentTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Create a new incident with the required information. \n" + self.short_description = "Create a new incident" + + +class NavigateAndCreateChangeRequestTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the change request list page and create a new change request. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new change request" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + task=CreateChangeRequestTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = ( + 'Create a new "Normal" change request with the required information. \n' + ) + self.short_description = 'Create a new "Normal" change request' + + +class NavigateAndCreateProblemTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the problem list page and create a new problem. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new problem" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Problem", + "module": "All", + }, + level=level, + task=CreateProblemTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Create a new problem with the required information. \n" + self.short_description = "Create a new problem" + + +class NavigateAndCreateHardwareAssetTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the hardware asset list page and create a new hardware asset. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new hardware asset" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + task=CreateHardwareAssetTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Create a new hardware asset with the required information. \n" + self.short_description = "Create a new hardware asset" + + +class NavigateAndOrderStandardLaptopTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order a standard laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a standard laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderStandardLaptopTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order a standard laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a standard laptop from the service catalog" + + +class NavigateAndOrderSalesLaptopTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order a sales laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a sales laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderSalesLaptopTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order a sales laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a sales laptop from the service catalog" + + +class NavigateAndOrderDeveloperLaptopTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order a developer laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a developer laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderDeveloperLaptopTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order a developer laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a developer laptop from the service catalog" + + +class NavigateAndOrderIpadProTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order an iPad Pro. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an iPad Pro" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderIpadProTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order an iPad Pro from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an iPad Pro from the service catalog" + + +class NavigateAndOrderIpadMiniTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order an iPad Mini. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an iPad Mini" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderIpadMiniTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order an iPad Mini from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an iPad Mini from the service catalog" + + +class NavigateAndOrderAppleWatchTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order an Apple Watch. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an Apple Watch" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderAppleWatchTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order an Apple Watch from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an Apple Watch from the service catalog" + + +class NavigateAndOrderAppleMacBookPro15Task(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order an Apple MacBook Pro 15". + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an Apple MacBook Pro 15" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderAppleMacBookPro15Task( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = 'Order an Apple MacBook Pro 15" from the service catalog with the required configuration if applicable. \n' + self.short_description = 'Order an Apple MacBook Pro 15" from the service catalog' + + +class NavigateAndOrderDevelopmentLaptopPCTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order a development laptop PC. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a development laptop PC" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderDevelopmentLaptopPCTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order a development laptop PC from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a development laptop PC from the service catalog" + + +class NavigateAndOrderLoanerLaptopTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Navigate to the service catalog item list page and order a loaner laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a loaner laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + task=OrderLoanerLaptopTask( + seed=seed, instance=instance, used_in_level_2=(level == 2), is_validated=True + ), + ) + self.task_description = "Order a loaner laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a loaner laptop from the service catalog" + + +class NavigateAndFilterAssetListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "alm_asset", + ) -> None: + """ + Navigate to the user list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > All Assets", + }, + level=level, + task=FilterAssetListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Filter the asset list - in Asset > Portflios > All Assets - based on specific criteria. \n" + self.short_description = "Filter the asset list." + self.list_name = list_name + + +class NavigateAndFilterUserListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "user", + ) -> None: + """ + Navigate to the user list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the user list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + task=FilterUserListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Filter the user list based on specific criteria. \n" + self.short_description = "Filter the user list." + self.list_name = list_name + + +class NavigateAndFilterIncidentListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "incident", + ) -> None: + """ + Navigate to the incident list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the incident list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + task=FilterIncidentListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Filter the incident list based on specific criteria. \n" + self.short_description = "Filter the incident list." + self.list_name = list_name + + +class NavigateAndFilterChangeRequestListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "change_request", + ) -> None: + """ + Navigate to the change request list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the change request list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + task=FilterChangeRequestListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Filter the change request list based on specific criteria. \n" + self.short_description = "Filter the change request list." + self.list_name = list_name + + +class NavigateAndFilterHardwareListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "hardware_asset", + ) -> None: + """ + Navigate to the hardware asset list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the hardware asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + task=FilterHardwareListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Filter the hardware asset list based on specific criteria. \n" + self.short_description = "Filter the hardware asset list." + self.list_name = list_name + + +class NavigateAndFilterServiceCatalogItemListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "service_catalog_item", + ) -> None: + """ + Navigate to the service catalog item list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the service catalog item list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Catalog", + "module": "Catalog Definitions > Maintain Items", + }, + level=level, + task=FilterServiceCatalogItemListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = ( + "Filter the service catalog item list based on specific criteria. \n" + ) + self.short_description = "Filter the service catalog item list." + self.list_name = list_name + + +class NavigateAndSortAssetListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "alm_asset", + ) -> None: + """ + Navigate to the user list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > All Assets", + }, + level=level, + task=SortAssetListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Sort the asset list - in Asset > Portflios > All Assets - based on specific criteria. \n" + self.short_description = "Sort the asset list." + self.list_name = list_name + + +class NavigateAndSortUserListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "user", + ) -> None: + """ + Navigate to the user list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the user list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + task=SortUserListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Sort the user list based on specific criteria. \n" + self.short_description = "Sort the user list." + self.list_name = list_name + + +class NavigateAndSortIncidentListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "incident", + ) -> None: + """ + Navigate to the incident list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the incident list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + task=SortIncidentListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Sort the incident list based on specific criteria. \n" + self.short_description = "Sort the incident list." + self.list_name = list_name + + +class NavigateAndSortChangeRequestListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "change_request", + ) -> None: + """ + Navigate to the change request list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the change request list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + task=SortChangeRequestListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Sort the change request list based on specific criteria. \n" + self.short_description = "Sort the change request list." + self.list_name = list_name + + +class NavigateAndSortHardwareListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "hardware_asset", + ) -> None: + """ + Navigate to the hardware asset list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the hardware asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + task=SortHardwareListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Sort the hardware asset list based on specific criteria. \n" + self.short_description = "Sort the hardware asset list." + self.list_name = list_name + + +class NavigateAndSortServiceCatalogItemListTask(NavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "service_catalog_item", + ) -> None: + """ + Navigate to the service catalog item list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the service catalog item list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Catalog", + "module": "Catalog Definitions > Maintain Items", + }, + level=level, + task=SortServiceCatalogItemListTask( + seed=seed, + instance=instance, + used_in_level_2=(level == 2), + is_validated=True, + list_name=list_name, + ), + ) + self.task_description = "Sort the service catalog item list based on specific criteria. \n" + self.short_description = "Sort the service catalog item list." + self.list_name = list_name + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) and issubclass(var, NavigateAndDoTask) and var is not NavigateAndDoTask +] + +NAVIGATE_AND_CREATE_TASKS = [ + NavigateAndCreateUserTask, + NavigateAndCreateIncidentTask, + NavigateAndCreateChangeRequestTask, + NavigateAndCreateProblemTask, + NavigateAndCreateHardwareAssetTask, +] +NAVIGATE_AND_ORDER_TASKS = [ + NavigateAndOrderStandardLaptopTask, + NavigateAndOrderSalesLaptopTask, + NavigateAndOrderDeveloperLaptopTask, + NavigateAndOrderIpadProTask, + NavigateAndOrderIpadMiniTask, + NavigateAndOrderAppleWatchTask, + NavigateAndOrderAppleMacBookPro15Task, + NavigateAndOrderDevelopmentLaptopPCTask, + NavigateAndOrderLoanerLaptopTask, +] +NAVIGATE_AND_FILTER_TASKS = [ + NavigateAndFilterAssetListTask, + NavigateAndFilterUserListTask, + NavigateAndFilterIncidentListTask, + NavigateAndFilterChangeRequestListTask, + NavigateAndFilterHardwareListTask, + NavigateAndFilterServiceCatalogItemListTask, +] +NAVIGATE_AND_SORT_TASKS = [ + NavigateAndSortAssetListTask, + NavigateAndSortUserListTask, + NavigateAndSortIncidentListTask, + NavigateAndSortChangeRequestListTask, + NavigateAndSortHardwareListTask, + NavigateAndSortServiceCatalogItemListTask, +] diff --git a/src/browsergym/workarena/tasks/compositional/navigate_and_do_infeasible.py b/src/browsergym/workarena/tasks/compositional/navigate_and_do_infeasible.py new file mode 100644 index 0000000..e6cf9e8 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/navigate_and_do_infeasible.py @@ -0,0 +1,2100 @@ +from faker import Faker + +fake = Faker() +from functools import partial + +from playwright.sync_api._generated import Page + +from browsergym.workarena.tasks.form import ( + CreateChangeRequestTask, + CreateHardwareAssetTask, + CreateIncidentTask, + CreateProblemTask, + CreateUserTask, +) +from browsergym.workarena.tasks.list import ( + FilterAssetListTask, + FilterChangeRequestListTask, + FilterHardwareListTask, + FilterIncidentListTask, + FilterServiceCatalogItemListTask, + FilterUserListTask, + SortAssetListTask, + SortChangeRequestListTask, + SortHardwareListTask, + SortIncidentListTask, + SortServiceCatalogItemListTask, + SortUserListTask, +) +from browsergym.workarena.tasks.navigation import AllMenuTask +from browsergym.workarena.tasks.service_catalog import ( + OrderDeveloperLaptopTask, + OrderIpadMiniTask, + OrderIpadProTask, + OrderSalesLaptopTask, + OrderStandardLaptopTask, + OrderAppleWatchTask, + OrderAppleMacBookPro15Task, + OrderDevelopmentLaptopPCTask, + OrderLoanerLaptopTask, +) + +from .base import HumanEvalTask, InfeasibleCompositionalTask +from .utils.infeasible_configs import ( + get_infeasible_form_config, + get_infeasible_service_catalog_config, + get_infeasible_filter_config, + get_infeasible_sort_config, +) + +from ..base import AbstractServiceNowTask + +from ...instance import SNowInstance + + +class InfeasibleNavigateAndDoTask(InfeasibleCompositionalTask, HumanEvalTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + function: callable = None, + provide_reason: bool = True, + navigation_config: dict = None, + level: int = 2, + task_class: AbstractServiceNowTask = None, + ) -> None: + """ + Generic task to navigate to a specific page and perform a task. + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of tuples, each containing a subtask + function: callable + Function that takes a valid config and renders it infeasible. + provide_reason: bool + Whether to provide a reason for the infeasibility. If False, the list of reasons will be [""] so that + any infeasibility can be detected by the absence of a reason. + navigation_config: dict + Configuration to use for the navigation task. Contains the application and the module; the URL is not necessary as the + nav step is not validated. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + task: AbstractServiceNowTask + The task to perform after navigating to the page. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. Provided by the child class. + short_description: str + A short description of the task to be completed. "Create a new user". Provided by the child class. + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + ) + self.used_in_level_2 = self.level == 2 + self.task_class = task_class + self.task_description = None + self.short_description = None + # Get the navigation configuration; there is only one configuration for each application and module combo + self.navigation_config = navigation_config + self.function = partial(function, provide_reason=provide_reason) + + def setup_goal(self, page: Page) -> tuple[str, dict]: + config = self.fixed_config if self.fixed_config else self._get_config() + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[AbstractServiceNowTask]: + valid_task_config = self.random.choice(self.task_class.all_configs()) + infeasible_task_config, self.infeasible_reasons = self.function( + config=valid_task_config, random=self.random + ) + config = [ + # Infeasible version of navigate to the task start page + AllMenuTask( + instance=self.instance, + fixed_config=self.navigation_config, + is_validated=False, + used_in_level_2=True, + has_description=True, + ), + self.task_class( + seed=self.seed, + instance=self.instance, + fixed_config=infeasible_task_config, + is_validated=False, + has_description=True, + used_in_level_2=self.used_in_level_2, + ), + ] + + return config + + +class InfeasibleNavigateAndCreateUserWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the user list page and create a new user. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new user" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + function=get_infeasible_form_config, + task_class=CreateUserTask, + provide_reason=True, + ) + self.task_description = "Create a new user with the required information. \n" + self.short_description = "Create a new user" + + +class InfeasibleNavigateAndCreateUserTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the user list page and create a new user. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new user" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + provide_reason=False, + function=get_infeasible_form_config, + task_class=CreateUserTask, + ) + self.task_description = "Create a new user with the required information. \n" + self.short_description = "Create a new user" + + +class InfeasibleNavigateAndCreateIncidentWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the incident list page and create a new incident. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new incident" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + function=get_infeasible_form_config, + task_class=CreateIncidentTask, + provide_reason=True, + ) + self.task_description = "Create a new incident with the required information. \n" + self.short_description = "Create a new incident" + + +class InfeasibleNavigateAndCreateIncidentTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the incident list page and create a new incident. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new incident" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + provide_reason=False, + function=get_infeasible_form_config, + task_class=CreateIncidentTask, + ) + self.task_description = "Create a new incident with the required information. \n" + self.short_description = "Create a new incident" + + +class InfeasibleNavigateAndCreateChangeRequestWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the change request list page and create a new change request. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new change request" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + function=get_infeasible_form_config, + task_class=CreateChangeRequestTask, + provide_reason=True, + ) + self.task_description = ( + 'Create a new "Normal" change request with the required information. \n' + ) + self.short_description = 'Create a new "Normal" change request' + + +class InfeasibleNavigateAndCreateChangeRequestTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the change request list page and create a new change request. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new change request" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + provide_reason=False, + function=get_infeasible_form_config, + task_class=CreateChangeRequestTask, + ) + self.task_description = ( + 'Create a new "Normal" change request with the required information. \n' + ) + self.short_description = 'Create a new "Normal" change request' + + +class InfeasibleNavigateAndCreateProblemWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the problem list page and create a new problem. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new problem" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Problem", + "module": "All", + }, + level=level, + function=get_infeasible_form_config, + task_class=CreateProblemTask, + provide_reason=True, + ) + self.task_description = "Create a new problem with the required information. \n" + self.short_description = "Create a new problem" + + +class InfeasibleNavigateAndCreateProblemTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the problem list page and create a new problem. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new problem" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Problem", + "module": "All", + }, + level=level, + provide_reason=False, + function=get_infeasible_form_config, + task_class=CreateProblemTask, + ) + self.task_description = "Create a new problem with the required information. \n" + self.short_description = "Create a new problem" + + +class InfeasibleNavigateAndCreateHardwareAssetWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the hardware asset list page and create a new hardware asset. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new hardware asset" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + function=get_infeasible_form_config, + task_class=CreateHardwareAssetTask, + provide_reason=True, + ) + self.task_description = "Create a new hardware asset with the required information. \n" + self.short_description = "Create a new hardware asset" + + +class InfeasibleNavigateAndCreateHardwareAssetTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the hardware asset list page and create a new hardware asset. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Create a new hardware asset" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + provide_reason=False, + function=get_infeasible_form_config, + task_class=CreateHardwareAssetTask, + ) + self.task_description = "Create a new hardware asset with the required information. \n" + self.short_description = "Create a new hardware asset" + + +class InfeasibleNavigateAndOrderStandardLaptopWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a standard laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a standard laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderStandardLaptopTask, + provide_reason=True, + ) + self.task_description = "Order a standard laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a standard laptop from the service catalog" + + +class InfeasibleNavigateAndOrderStandardLaptopTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a standard laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a standard laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderStandardLaptopTask, + ) + self.task_description = "Order a standard laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a standard laptop from the service catalog" + + +class InfeasibleNavigateAndOrderSalesLaptopWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a sales laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a sales laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderSalesLaptopTask, + provide_reason=True, + ) + self.task_description = "Order a sales laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a sales laptop from the service catalog" + + +class InfeasibleNavigateAndOrderSalesLaptopTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a sales laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a sales laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderSalesLaptopTask, + ) + self.task_description = "Order a sales laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a sales laptop from the service catalog" + + +class InfeasibleNavigateAndOrderDeveloperLaptopWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a developer laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a developer laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderDeveloperLaptopTask, + provide_reason=True, + ) + self.task_description = "Order a developer laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a developer laptop from the service catalog" + + +class InfeasibleNavigateAndOrderDeveloperLaptopTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a developer laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a developer laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderDeveloperLaptopTask, + ) + self.task_description = "Order a developer laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a developer laptop from the service catalog" + + +class InfeasibleNavigateAndOrderIpadProWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an iPad Pro. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an iPad Pro" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderIpadProTask, + provide_reason=True, + ) + self.task_description = "Order an iPad Pro from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an iPad Pro from the service catalog" + + +class InfeasibleNavigateAndOrderIpadProTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an iPad Pro. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an iPad Pro" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderIpadProTask, + ) + self.task_description = "Order an iPad Pro from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an iPad Pro from the service catalog" + + +class InfeasibleNavigateAndOrderIpadMiniWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an iPad Mini. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an iPad Mini" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderIpadMiniTask, + provide_reason=True, + ) + self.task_description = "Order an iPad Mini from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an iPad Mini from the service catalog" + + +class InfeasibleNavigateAndOrderIpadMiniTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an iPad Mini. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an iPad Mini" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderIpadMiniTask, + ) + self.task_description = "Order an iPad Mini from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an iPad Mini from the service catalog" + + +class InfeasibleNavigateAndOrderAppleWatchWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an Apple Watch. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an Apple Watch" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderAppleWatchTask, + provide_reason=True, + ) + self.task_description = "Order an Apple Watch from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an Apple Watch from the service catalog" + + +class InfeasibleNavigateAndOrderAppleWatchTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an Apple Watch. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an Apple Watch" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderAppleWatchTask, + ) + self.task_description = "Order an Apple Watch from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order an Apple Watch from the service catalog" + + +class InfeasibleNavigateAndOrderAppleMacBookPro15WithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an Apple MacBook Pro 15". + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an Apple MacBook Pro 15" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderAppleMacBookPro15Task, + provide_reason=True, + ) + self.task_description = 'Order an Apple MacBook Pro 15" from the service catalog with the required configuration if applicable. \n' + self.short_description = 'Order an Apple MacBook Pro 15" from the service catalog' + + +class InfeasibleNavigateAndOrderAppleMacBookPro15Task(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order an Apple MacBook Pro 15". + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order an Apple MacBook Pro 15" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderAppleMacBookPro15Task, + ) + self.task_description = 'Order an Apple MacBook Pro 15" from the service catalog with the required configuration if applicable. \n' + self.short_description = 'Order an Apple MacBook Pro 15" from the service catalog' + + +class InfeasibleNavigateAndOrderDevelopmentLaptopPCWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a development laptop PC. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a development laptop PC" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderDevelopmentLaptopPCTask, + provide_reason=True, + ) + self.task_description = "Order a development laptop PC from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a development laptop PC from the service catalog" + + +class InfeasibleNavigateAndOrderDevelopmentLaptopPCTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a development laptop PC. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a development laptop PC" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderDevelopmentLaptopPCTask, + ) + self.task_description = "Order a development laptop PC from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a development laptop PC from the service catalog" + + +class InfeasibleNavigateAndOrderLoanerLaptopWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a loaner laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a loaner laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + function=get_infeasible_service_catalog_config, + task_class=OrderLoanerLaptopTask, + provide_reason=True, + ) + self.task_description = "Order a loaner laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a loaner laptop from the service catalog" + + +class InfeasibleNavigateAndOrderLoanerLaptopTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and order a loaner laptop. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Order a loaner laptop" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Self-Service", + "module": "Service Catalog", + }, + level=level, + provide_reason=False, + function=get_infeasible_service_catalog_config, + task_class=OrderLoanerLaptopTask, + ) + self.task_description = "Order a loaner laptop from the service catalog with the required configuration if applicable. \n" + self.short_description = "Order a loaner laptop from the service catalog" + + +class InfeasibleNavigateAndFilterAssetListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "alm_asset", + ) -> None: + """ + Infeasible version of navigate to the user list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > All Assets", + }, + level=level, + function=get_infeasible_filter_config, + task_class=FilterAssetListTask, + provide_reason=True, + ) + self.task_description = "Filter the asset list - in Asset > Portflios > All Assets - based on specific criteria. \n" + self.short_description = "Filter the asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterAssetListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "alm_asset", + ) -> None: + """ + Infeasible version of navigate to the user list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > All Assets", + }, + level=level, + provide_reason=False, + function=get_infeasible_filter_config, + task_class=FilterAssetListTask, + ) + self.task_description = "Filter the asset list - in Asset > Portflios > All Assets - based on specific criteria. \n" + self.short_description = "Filter the asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterUserListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "user", + ) -> None: + """ + Infeasible version of navigate to the user list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the user list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + function=get_infeasible_filter_config, + task_class=FilterUserListTask, + provide_reason=True, + ) + self.task_description = "Filter the user list based on specific criteria. \n" + self.short_description = "Filter the user list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterUserListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "user", + ) -> None: + """ + Infeasible version of navigate to the user list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the user list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + provide_reason=False, + function=get_infeasible_filter_config, + task_class=FilterUserListTask, + ) + self.task_description = "Filter the user list based on specific criteria. \n" + self.short_description = "Filter the user list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterIncidentListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "incident", + ) -> None: + """ + Infeasible version of navigate to the incident list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the incident list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + function=get_infeasible_filter_config, + task_class=FilterIncidentListTask, + provide_reason=True, + ) + self.task_description = "Filter the incident list based on specific criteria. \n" + self.short_description = "Filter the incident list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterIncidentListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "incident", + ) -> None: + """ + Infeasible version of navigate to the incident list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the incident list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + provide_reason=False, + function=get_infeasible_filter_config, + task_class=FilterIncidentListTask, + ) + self.task_description = "Filter the incident list based on specific criteria. \n" + self.short_description = "Filter the incident list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterChangeRequestListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "change_request", + ) -> None: + """ + Infeasible version of navigate to the change request list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the change request list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + function=get_infeasible_filter_config, + task_class=FilterChangeRequestListTask, + provide_reason=True, + ) + self.task_description = "Filter the change request list based on specific criteria. \n" + self.short_description = "Filter the change request list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterChangeRequestListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "change_request", + ) -> None: + """ + Infeasible version of navigate to the change request list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the change request list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + provide_reason=False, + function=get_infeasible_filter_config, + task_class=FilterChangeRequestListTask, + ) + self.task_description = "Filter the change request list based on specific criteria. \n" + self.short_description = "Filter the change request list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterHardwareListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "hardware_asset", + ) -> None: + """ + Infeasible version of navigate to the hardware asset list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the hardware asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + function=get_infeasible_filter_config, + task_class=FilterHardwareListTask, + provide_reason=True, + ) + self.task_description = "Filter the hardware asset list based on specific criteria. \n" + self.short_description = "Filter the hardware asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterHardwareListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "hardware_asset", + ) -> None: + """ + Infeasible version of navigate to the hardware asset list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the hardware asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + provide_reason=False, + function=get_infeasible_filter_config, + task_class=FilterHardwareListTask, + ) + self.task_description = "Filter the hardware asset list based on specific criteria. \n" + self.short_description = "Filter the hardware asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterServiceCatalogItemListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "service_catalog_item", + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the service catalog item list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Catalog", + "module": "Catalog Definitions > Maintain Items", + }, + level=level, + function=get_infeasible_filter_config, + task_class=FilterServiceCatalogItemListTask, + provide_reason=True, + ) + self.task_description = ( + "Filter the service catalog item list based on specific criteria. \n" + ) + self.short_description = "Filter the service catalog item list." + self.list_name = list_name + + +class InfeasibleNavigateAndFilterServiceCatalogItemListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "service_catalog_item", + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and filter the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the service catalog item list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Catalog", + "module": "Catalog Definitions > Maintain Items", + }, + level=level, + provide_reason=False, + function=get_infeasible_filter_config, + task_class=FilterServiceCatalogItemListTask, + ) + self.task_description = ( + "Filter the service catalog item list based on specific criteria. \n" + ) + self.short_description = "Filter the service catalog item list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortAssetListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "alm_asset", + ) -> None: + """ + Infeasible version of navigate to the user list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > All Assets", + }, + level=level, + function=get_infeasible_sort_config, + task_class=SortAssetListTask, + provide_reason=True, + ) + self.task_description = "Sort the asset list - in Asset > Portflios > All Assets - based on specific criteria. \n" + self.short_description = "Sort the asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortAssetListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "alm_asset", + ) -> None: + """ + Infeasible version of navigate to the user list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Filter the asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > All Assets", + }, + level=level, + provide_reason=False, + function=get_infeasible_sort_config, + task_class=SortAssetListTask, + ) + self.task_description = "Sort the asset list - in Asset > Portflios > All Assets - based on specific criteria. \n" + self.short_description = "Sort the asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortUserListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "user", + ) -> None: + """ + Infeasible version of navigate to the user list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the user list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + function=get_infeasible_sort_config, + task_class=SortUserListTask, + provide_reason=True, + ) + self.task_description = "Sort the user list based on specific criteria. \n" + self.short_description = "Sort the user list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortUserListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "user", + ) -> None: + """ + Infeasible version of navigate to the user list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the user list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Organization", + "module": "Users", + }, + level=level, + provide_reason=False, + function=get_infeasible_sort_config, + task_class=SortUserListTask, + ) + self.task_description = "Sort the user list based on specific criteria. \n" + self.short_description = "Sort the user list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortIncidentListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "incident", + ) -> None: + """ + Infeasible version of navigate to the incident list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the incident list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + function=get_infeasible_sort_config, + task_class=SortIncidentListTask, + provide_reason=True, + ) + self.task_description = "Sort the incident list based on specific criteria. \n" + self.short_description = "Sort the incident list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortIncidentListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "incident", + ) -> None: + """ + Infeasible version of navigate to the incident list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the incident list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Desk", + "module": "Incidents", + }, + level=level, + provide_reason=False, + function=get_infeasible_sort_config, + task_class=SortIncidentListTask, + ) + self.task_description = "Sort the incident list based on specific criteria. \n" + self.short_description = "Sort the incident list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortChangeRequestListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "change_request", + ) -> None: + """ + Infeasible version of navigate to the change request list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the change request list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + function=get_infeasible_sort_config, + task_class=SortChangeRequestListTask, + provide_reason=True, + ) + self.task_description = "Sort the change request list based on specific criteria. \n" + self.short_description = "Sort the change request list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortChangeRequestListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "change_request", + ) -> None: + """ + Infeasible version of navigate to the change request list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the change request list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Change", + "module": "All", + }, + level=level, + provide_reason=False, + function=get_infeasible_sort_config, + task_class=SortChangeRequestListTask, + ) + self.task_description = "Sort the change request list based on specific criteria. \n" + self.short_description = "Sort the change request list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortHardwareListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "hardware_asset", + ) -> None: + """ + Infeasible version of navigate to the hardware asset list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the hardware asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + function=get_infeasible_sort_config, + task_class=SortHardwareListTask, + provide_reason=True, + ) + self.task_description = "Sort the hardware asset list based on specific criteria. \n" + self.short_description = "Sort the hardware asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortHardwareListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "hardware_asset", + ) -> None: + """ + Infeasible version of navigate to the hardware asset list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the hardware asset list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + }, + level=level, + provide_reason=False, + function=get_infeasible_sort_config, + task_class=SortHardwareListTask, + ) + self.task_description = "Sort the hardware asset list based on specific criteria. \n" + self.short_description = "Sort the hardware asset list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortServiceCatalogItemListWithReasonTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "service_catalog_item", + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the service catalog item list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Catalog", + "module": "Catalog Definitions > Maintain Items", + }, + level=level, + function=get_infeasible_sort_config, + task_class=SortServiceCatalogItemListTask, + provide_reason=True, + ) + self.task_description = "Sort the service catalog item list based on specific criteria. \n" + self.short_description = "Sort the service catalog item list." + self.list_name = list_name + + +class InfeasibleNavigateAndSortServiceCatalogItemListTask(InfeasibleNavigateAndDoTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + list_name: str = "service_catalog_item", + ) -> None: + """ + Infeasible version of navigate to the service catalog item list page and sort the list based on a specific criteria. + + Attributes: + ----------- + task_description: str + The start of the task description to be completed. + short_description: str + A short description of the task to be completed. "Sort the service catalog item list" + """ + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + navigation_config={ + "application": "Service Catalog", + "module": "Catalog Definitions > Maintain Items", + }, + level=level, + provide_reason=False, + function=get_infeasible_sort_config, + task_class=SortServiceCatalogItemListTask, + ) + self.task_description = "Sort the service catalog item list based on specific criteria. \n" + self.short_description = "Sort the service catalog item list." + self.list_name = list_name + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) + and issubclass(var, InfeasibleNavigateAndDoTask) + and var is not InfeasibleNavigateAndDoTask +] + +INFEASIBLE_NAVIGATE_AND_CREATE_WITH_REASON = [ + InfeasibleNavigateAndCreateUserWithReasonTask, + InfeasibleNavigateAndCreateIncidentWithReasonTask, + InfeasibleNavigateAndCreateChangeRequestWithReasonTask, + InfeasibleNavigateAndCreateProblemWithReasonTask, + InfeasibleNavigateAndCreateHardwareAssetWithReasonTask, +] +INFEASIBLE_NAVIGATE_AND_CREATE = [ + InfeasibleNavigateAndCreateUserTask, + InfeasibleNavigateAndCreateIncidentTask, + InfeasibleNavigateAndCreateChangeRequestTask, + InfeasibleNavigateAndCreateProblemTask, + InfeasibleNavigateAndCreateHardwareAssetTask, +] +INFEASIBLE_NAVIGATE_AND_ORDER_WITH_REASON = [ + InfeasibleNavigateAndOrderStandardLaptopWithReasonTask, + InfeasibleNavigateAndOrderSalesLaptopWithReasonTask, + InfeasibleNavigateAndOrderDeveloperLaptopWithReasonTask, + InfeasibleNavigateAndOrderIpadProWithReasonTask, + InfeasibleNavigateAndOrderIpadMiniWithReasonTask, + InfeasibleNavigateAndOrderAppleWatchWithReasonTask, + InfeasibleNavigateAndOrderAppleMacBookPro15WithReasonTask, + InfeasibleNavigateAndOrderDevelopmentLaptopPCWithReasonTask, + InfeasibleNavigateAndOrderLoanerLaptopWithReasonTask, +] +INFEASIBLE_NAVIGATE_AND_ORDER = [ + InfeasibleNavigateAndOrderStandardLaptopTask, + InfeasibleNavigateAndOrderSalesLaptopTask, + InfeasibleNavigateAndOrderDeveloperLaptopTask, + InfeasibleNavigateAndOrderIpadProTask, + InfeasibleNavigateAndOrderIpadMiniTask, + InfeasibleNavigateAndOrderAppleWatchTask, + InfeasibleNavigateAndOrderAppleMacBookPro15Task, + InfeasibleNavigateAndOrderDevelopmentLaptopPCTask, + InfeasibleNavigateAndOrderLoanerLaptopTask, +] +INFEASIBLE_NAVIGATE_AND_FILTER_WITH_REASON = [ + InfeasibleNavigateAndFilterAssetListWithReasonTask, + InfeasibleNavigateAndFilterUserListWithReasonTask, + InfeasibleNavigateAndFilterIncidentListWithReasonTask, + InfeasibleNavigateAndFilterChangeRequestListWithReasonTask, + InfeasibleNavigateAndFilterHardwareListWithReasonTask, + InfeasibleNavigateAndFilterServiceCatalogItemListWithReasonTask, +] +INFEASIBLE_NAVIGATE_AND_FILTER = [ + InfeasibleNavigateAndFilterAssetListTask, + InfeasibleNavigateAndFilterUserListTask, + InfeasibleNavigateAndFilterIncidentListTask, + InfeasibleNavigateAndFilterChangeRequestListTask, + InfeasibleNavigateAndFilterHardwareListTask, + InfeasibleNavigateAndFilterServiceCatalogItemListTask, +] +INFEASIBLE_NAVIGATE_AND_SORT_WITH_REASON = [ + InfeasibleNavigateAndSortAssetListWithReasonTask, + InfeasibleNavigateAndSortUserListWithReasonTask, + InfeasibleNavigateAndSortIncidentListWithReasonTask, + InfeasibleNavigateAndSortChangeRequestListWithReasonTask, + InfeasibleNavigateAndSortHardwareListWithReasonTask, + InfeasibleNavigateAndSortServiceCatalogItemListWithReasonTask, +] +INFEASIBLE_NAVIGATE_AND_SORT = [ + InfeasibleNavigateAndSortAssetListTask, + InfeasibleNavigateAndSortUserListTask, + InfeasibleNavigateAndSortIncidentListTask, + InfeasibleNavigateAndSortChangeRequestListTask, + InfeasibleNavigateAndSortHardwareListTask, + InfeasibleNavigateAndSortServiceCatalogItemListTask, +] diff --git a/src/browsergym/workarena/tasks/compositional/offboard_user.py b/src/browsergym/workarena/tasks/compositional/offboard_user.py new file mode 100644 index 0000000..b24f0e9 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/offboard_user.py @@ -0,0 +1,207 @@ +import json + +from faker import Faker + +fake = Faker() +from playwright.sync_api._generated import Page + +from .base import CompositionalTask, HumanEvalTask +from .delete_record import DeleteUserTask + +from ..base import AbstractServiceNowTask +from ..form import EditHardwareAssetTask +from ..knowledge import KnowledgeBaseSearchTask +from ..list import FilterHardwareListTask +from ..navigation import AllMenuTask + +from ...api.computer_asset import create_computer_asset +from ...api.user import create_user +from ...api.utils import table_api_call, db_delete_from_table +from ...instance import SNowInstance + + +class OffBoardUserTask(CompositionalTask, HumanEvalTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Employee OffBoarding Task + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of subtasks. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. e.g. "Referring to company protocol 'Offboarding a user', offboard user XYZ" + short_description: str + A short description of the task to be completed. e.g. "Offboard user John Doe" + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + self.protocol_name = "Offboarding a user" + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + protocol_name=self.protocol_name, + ) + self.task_description = None + self.short_description = None + self.user_full_name = None + self.user_sys_id = None + self.user_name = None + self.laptop_asset_tag = None + self.laptop_sys_id = None + + def setup_goal(self, page: Page) -> tuple[str, dict]: + # Generate random name for the user + first_name = fake.first_name() + "-" + fake.first_name() + last_name = fake.last_name() + "-" + fake.last_name() + self.user_full_name = first_name + " " + last_name + self.laptop_asset_tag = "P" + str(id(self) % (10**8)).zfill(8) + + # Create user + self.user_name, _, self.user_sys_id = create_user( + instance=self.instance, first_name=first_name, last_name=last_name, random=self.random + ) + + assert self.user_sys_id, f"Failed to create user {first_name} {last_name}" + + self.laptop_sys_id, _, _ = create_computer_asset( + instance=self.instance, + asset_tag=self.laptop_asset_tag, + user_sys_id=self.user_sys_id, + random=self.random, + ) + + config = self.fixed_config if self.fixed_config else self._get_config() + # Get the task description + self.short_description = f"Offboard user {self.user_full_name}" + self.task_description = f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) offboard user "{self.user_full_name}" \n' + + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[AbstractServiceNowTask]: + """Sample a user configuration and a hardware asset configuration. Add the assigned_to field if missing + from the hardware asset configuration. Finally, return the list of subtasks, with navigation subtasks included. + """ + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": f'Can you find the "{self.protocol_name}" Protocol in the Knowledge Base?', + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + unassign_hardware_subtask = [ + # Navigate to the hardware asset list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + "url": "/now/nav/ui/classic/params/target/alm_hardware_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + FilterHardwareListTask( + instance=self.instance, + fixed_config={ + "filter_columns": ["assigned_to"], + "filter_kind": "AND", + "filter_values": [f"{self.user_full_name}"], + }, + is_validated=False, + used_in_level_2=True, + ), + # Create a new hardware asset + EditHardwareAssetTask( + instance=self.instance, + record_sys_id=self.laptop_sys_id, + new_values={"assigned_to": ""}, + is_validated=True, + used_in_level_2=True, + level=self.level, + ), + ] + delete_user_subtask = [ + # Navigate to the user list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "System Security", + "module": "Users and Groups > Users", + "url": "/now/nav/ui/classic/params/target/sys_user_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create a new user + DeleteUserTask( + instance=self.instance, + fixed_config={ + "field_name": "name", + "pretty_printed_field_name": "Name", + "field_value": self.user_full_name, + "other_fields": {}, + }, + record_sys_id=self.user_sys_id, + is_validated=True, + used_in_level_2=True, + ), + ] + + config = navigate_to_protocol_subtask + unassign_hardware_subtask + delete_user_subtask + + return config + + def teardown(self) -> None: + # Delete the user + user_record = table_api_call( + instance=self.instance, + table="sys_user", + params={"sysparm_query": f"sys_id={self.user_sys_id}"}, + )["result"] + if user_record: + db_delete_from_table( + instance=self.instance, + table="sys_user", + sys_id=self.user_sys_id, + ) + super().teardown() + + +__TASKS__ = [OffBoardUserTask] diff --git a/src/browsergym/workarena/tasks/compositional/onboard_user.py b/src/browsergym/workarena/tasks/compositional/onboard_user.py new file mode 100644 index 0000000..531cece --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/onboard_user.py @@ -0,0 +1,226 @@ +import json + +from playwright.sync_api._generated import Page + +from .base import CompositionalTask, HumanEvalTask + +from ..base import AbstractServiceNowTask +from ..form import CreateUserTask, CreateHardwareAssetTask +from ..knowledge import KnowledgeBaseSearchTask +from ..navigation import AllMenuTask +from ..service_catalog import OrderAppleMacBookPro15Task + +from ...instance import SNowInstance +from ...config import CREATE_USER_CONFIG_PATH, CREATE_HARDWARE_CONFIG_PATH + + +class OnBoardUserTask(CompositionalTask, HumanEvalTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Create a compositional task with specific subtasks + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of subtasks. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. e.g. "Referring to company protocol 'Onboarding a new user', onboard user with the following information: \n" + short_description: str + A short description of the task to be completed. e.g. "Onboard user John Doe" + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + self.protocol_name = "Onboarding a new user" + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + protocol_name=self.protocol_name, + ) + + self.all_user_configs = CreateUserTask.all_configs() + self.all_hardware_asset_configs = CreateHardwareAssetTask.all_configs() + self.task_description = None + self.short_description = None + + def setup_goal(self, page: Page) -> tuple[str, dict]: + # Sample a configuration + config = self.fixed_config if self.fixed_config else self._get_config() + user_name = ( + config[3].fixed_config["template_record"]["first_name"] + + " " + + config[3].fixed_config["template_record"]["last_name"] + ) + # Get the task description + self.short_description = f"Onboard user {user_name}" + self.task_description = f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) onboard user with the following information: \n' + + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[AbstractServiceNowTask]: + # Sample base configurations; the hardware config will be modified to include the assigned_to field + user_config = self.random.choice(self.all_user_configs) + hardware_config = self.random.choice(self.all_hardware_asset_configs) + + # Get the common fields between the user and hardware configurations to adjust the hardware config + common_fields = [ + field for field in hardware_config["fields"].keys() if field in user_config["fields"] + ] + common_task_fields = [ + field for field in hardware_config["task_fields"] if field in user_config["task_fields"] + ] + common_template_record_fields = [ + field + for field in hardware_config["template_record"].keys() + if field in user_config["template_record"] and "sys" not in field + ] + + # Drop the common fields as they create synchronization issues + for field in common_fields + common_task_fields + common_template_record_fields: + if field in user_config["fields"]: + user_config["fields"].pop(field) + if field in hardware_config["fields"]: + hardware_config["fields"].pop(field) + + if field in user_config["task_fields"]: + user_config["task_fields"].remove(field) + if field in hardware_config["task_fields"]: + hardware_config["task_fields"].remove(field) + + if field in user_config["template_record"]: + user_config["template_record"].pop(field) + if field in hardware_config["template_record"]: + hardware_config["template_record"].pop(field) + + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": f"Can you find the '{self.protocol_name}' Protocol in the Knowledge Base?", + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + + create_user_subtask = [ + # Navigate to the user list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "System Security", + "module": "Users and Groups > Users", + "url": "/now/nav/ui/classic/params/target/sys_user_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create a new user + CreateUserTask( + instance=self.instance, + fixed_config=user_config, + is_validated=True, + used_in_level_2=True, + ), + ] + + order_hardware_subtask = [ + # Navigate to the hardware asset list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Service Catalog", + "url": "/now/nav/ui/classic/params/target/catalog_home.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Order a MacBook Pro 15 + OrderAppleMacBookPro15Task( + instance=self.instance, + fixed_config={ + "configuration": {}, + "description": "Apple MacBook Pro", + "item": "Apple MacBook Pro 15", + "quantity": 1, + }, + is_validated=True, + used_in_level_2=True, + ), + ] + # The unique name for the user is created once the task is instantiated + user_full_name = ( + create_user_subtask[1].template_record["first_name"] + + " " + + create_user_subtask[1].template_record["last_name"] + ) + # Set the assigned_to field in the hardware asset configuration to the user's email + hardware_config["template_record"]["assigned_to"] = user_full_name + if "assigned_to" not in hardware_config["task_fields"]: + hardware_config["task_fields"].append("assigned_to") + + create_hardware_subtask = [ + # Navigate to the hardware asset list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + "url": "/now/nav/ui/classic/params/target/alm_hardware_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Create a new hardware asset + CreateHardwareAssetTask( + instance=self.instance, + fixed_config=hardware_config, + is_validated=True, + used_in_level_2=True, + ), + ] + + config = ( + navigate_to_protocol_subtask + + create_user_subtask + + order_hardware_subtask + + create_hardware_subtask + ) + + return config + + +__TASKS__ = [OnBoardUserTask] diff --git a/src/browsergym/workarena/tasks/compositional/update_task.py b/src/browsergym/workarena/tasks/compositional/update_task.py new file mode 100644 index 0000000..2a8e3d8 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/update_task.py @@ -0,0 +1,145 @@ +from playwright.sync_api import Page +from typing import List, Tuple + +from ..base import AbstractServiceNowTask +from ..comp_building_block import CompositionalBuildingBlockTask +from ..utils.utils import check_url_suffix_match +from ..utils.private_tasks import create_private_task_and_get_sys_id + +from ...api.utils import db_delete_from_table, table_api_call + + +class UpdatePrivateTask(AbstractServiceNowTask, CompositionalBuildingBlockTask): + """ + Set a private task to complete, assuming we start on the task viewed as form. + + Parameters: + ----------- + instance: SNowInstance + The instance to use. + start_rel_url: str + The relative URL of the task list. + fixed_config: dict + Configuration to use for the task. If provided, the task will use the provided configuration instead of + selecting a random one. See browsergym/workarena/data_files/task_configs/filter_change_request_list_task.json + for an example of a configuration file. + set_as_completed: bool + Whether the task should be marked as complete or not. If True, the task will be marked as complete; otherwise, marked as. + used to set infeasible tasks to incomplete. + """ + + def __init__( + self, + seed: int = None, + instance=None, + start_rel_url="/now/nav/ui/classic/params/target/task_list.do%3Fsysparm_userpref_module%3D1523b8d4c611227b00be8216ec331b9a%26sysparm_query%3Dactive%253Dtrue%255Eassigned_to%253Djavascript%253AgetMyAssignments%2528%2529%255Estate%2521%253D-5%255EEQ", + fixed_config: dict = None, + set_as_completed: bool = True, + **kwargs, + ) -> None: + super().__init__(seed=seed, instance=instance, start_rel_url=start_rel_url) + self.fixed_config = fixed_config + self.config = fixed_config + self.set_as_completed = set_as_completed + # 3 is the state for "Closed-Complete", 4 is "Closed-Incomplete", 7 is "Closed-Skipped" + self.allowed_options = ["3"] if self.set_as_completed else ["4", "7"] + self.private_task_id = "PTSK" + str(id(self) % (10**8)).zfill(8) + if self.fixed_config is None: + self.config = { + "task_description": "Close private task", + "short_description": self.private_task_id, + } + self.sys_id = None + self.task_rel_url = None # Relative URL of the task in form view + self.__dict__.update(kwargs) + + def setup_goal(self, page: Page) -> tuple[str, dict]: + task_description = self.config["task_description"] + short_description = self.config["short_description"] + self.sys_id = create_private_task_and_get_sys_id( + self.instance, + page, + self.private_task_id, + task_description, + short_description, + user_sys_id=self._base_user_sysid, + ) + self.task_rel_url = ( + f"/now/nav/ui/classic/params/target/vtb_task.do%3Fsys_id%3D{self.sys_id}" + ) + goal = f"Close private task {self.private_task_id}" + + return goal, {} + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + task_info = "Don't forget to mark this task as complete once you're done." + + return task_info + + def cheat(self, page: Page, chat_messages: list[str]) -> None: + super().cheat(page, chat_messages) + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + # Search for the private task by search for the number + frame.get_by_label("Search a specific field of the Tasks list").select_option("number") + search_input = frame.locator('input[aria-label="Search"]') + search_input.click() + search_input.fill(self.private_task_id) + search_input.press("Enter") + page.wait_for_timeout(1500) + # Click on the private task to open it + frame.get_by_label(f"Open record: {self.private_task_id}").click() + page.wait_for_timeout(2000) + page.wait_for_load_state("networkidle") + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + page.wait_for_timeout(1500) + # Click on the task state, select "Closed-Complete" if complete, else "Closed Skipped" and update the task + option = "3" if self.set_as_completed else "7" + frame.get_by_label("state").first.select_option(option) + frame.get_by_text("update").first.click() + # Wait for record to be updated in the DB + record_updated = False + while not record_updated: + record = table_api_call( + instance=self.instance, + table="vtb_task", + params={"sysparm_query": f"task_effective_number={self.private_task_id}"}, + )["result"] + record_updated = record[0]["state"] == option + page.wait_for_timeout(1000) + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + """ + Validate the solution + """ + record = table_api_call( + instance=self.instance, + table="vtb_task", + params={"sysparm_query": f"task_effective_number={self.private_task_id}"}, + )["result"] + if not record: + return 0, False, "", {"message": "Private task not found."} + if record[0]["state"] not in self.allowed_options: + return 0, False, "", {"message": "Private task not closed appropriately."} + + return 1, True, "Nice work, thank you!", {"message": "Private task was closed."} + + def teardown(self) -> None: + record_exists = table_api_call( + instance=self.instance, + table="vtb_task", + params={"sysparm_query": f"sys_id={self.sys_id}"}, + )["result"] + if record_exists: + db_delete_from_table( + instance=self.instance, + table="vtb_task", + sys_id=self.sys_id, + ) + super().teardown() + + +__TASKS__ = [UpdatePrivateTask] diff --git a/src/browsergym/workarena/tasks/compositional/utils/curriculum.py b/src/browsergym/workarena/tasks/compositional/utils/curriculum.py new file mode 100644 index 0000000..ce1755e --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/utils/curriculum.py @@ -0,0 +1,215 @@ +# from .edit_knowledge_base import __TASKS__ as EDIT_KNOWLEDGE_BASE_TASKS, __L2_TASKS__ as EDIT_KNOWLEDGE_BASE_L2_TASKS, __L3_TASKS__ as EDIT_KNOWLEDGE_BASE_L3TASKS +from ..dash_do_catalog import ( + DASH_AND_ORDER, + DASH_COMPUTE_MEAN_AND_ORDER, + DASH_COMPUTE_MEDIAN_AND_ORDER, + DASH_COMPUTE_MODE_AND_ORDER, +) +from ..dash_do_create_incident import DASH_AND_CREATE_INCIDENT, DASH_COMPUTE_AND_CREATE_INCIDENT +from ..dash_do_create_problem import DASH_AND_CREATE_PROBLEM, DASH_COMPUTE_AND_CREATE_PROBLEM +from ..dash_do_filter import ( + DASH_COMPUTE_MIN_FILTER_LIST, + DASH_COMPUTE_MAX_FILTER_LIST, + DASH_COMPUTE_MEAN_FILTER_LIST, + DASH_COMPUTE_MEDIAN_FILTER_LIST, + DASH_COMPUTE_MODE_FILTER_LIST, +) +from ..dash_do_request_item import ( + DASH_AND_REQUEST, + DASH_COMPUTE_MEAN_AND_REQUEST, + DASH_COMPUTE_MEDIAN_AND_REQUEST, + DASH_COMPUTE_MODE_AND_REQUEST, +) +from ..expense_management import __TASKS__ as EXPENSE_MANAGEMENT_TASKS +from ..find_and_order_item import __TASKS__ as FIND_AND_ORDER_ITEM_TASKS +from ..manage_change_request_schedule import ( + SMALL_BASE_SCHEDULING_TASKS, + LARGE_BASE_SCHEDULING_TASKS, + SMALL_TIGHT_SCHEDULING_TASKS, + LARGE_TIGHT_SCHEDULING_TASKS, +) +from ..mark_duplicate_problems import __TASKS__ as MARK_DUPLICATE_PROBLEMS_TASKS +from ..maximize_investment_return import __TASKS__ as MAXIMIZE_INVESTMENT_RETURN_TASKS +from ..navigate_and_do import ( + NAVIGATE_AND_CREATE_TASKS, + NAVIGATE_AND_FILTER_TASKS, + NAVIGATE_AND_ORDER_TASKS, + NAVIGATE_AND_SORT_TASKS, +) +from ..navigate_and_do_infeasible import ( + INFEASIBLE_NAVIGATE_AND_CREATE_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_CREATE, + INFEASIBLE_NAVIGATE_AND_ORDER_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_ORDER, + INFEASIBLE_NAVIGATE_AND_FILTER_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_FILTER, + INFEASIBLE_NAVIGATE_AND_SORT_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_SORT, +) +from ..offboard_user import __TASKS__ as OFFBOARD_USER_TASKS +from ..onboard_user import __TASKS__ as ONBOARD_USER_TASKS +from ..warranty_check import __TASKS__ as WARRANTY_CHECK_TASKS +from ..work_assignment import __TASKS__ as WORK_ASSIGNMENT_TASKS +from ..workload_balancing import __TASKS__ as WORKLOAD_BALANCING_TASKS + +AGENT_CURRICULUM = { + "planning_and_problem_solving": { + "buckets": [ + MARK_DUPLICATE_PROBLEMS_TASKS, + WORKLOAD_BALANCING_TASKS, + WORK_ASSIGNMENT_TASKS, + SMALL_BASE_SCHEDULING_TASKS, + LARGE_BASE_SCHEDULING_TASKS, + SMALL_TIGHT_SCHEDULING_TASKS, + LARGE_TIGHT_SCHEDULING_TASKS, + ], + "num_seeds": 2, + "weights": [9, 3, 6, 1, 1, 1, 1], + }, + "information_retrieval": { + "buckets": [ + DASH_AND_ORDER, + DASH_AND_CREATE_INCIDENT, + DASH_AND_CREATE_PROBLEM, + DASH_COMPUTE_MIN_FILTER_LIST, + DASH_COMPUTE_MAX_FILTER_LIST, + DASH_AND_REQUEST, + WARRANTY_CHECK_TASKS, + FIND_AND_ORDER_ITEM_TASKS, + ], + "num_seeds": 7, + "weights": [1, 1, 1, 1, 1, 1, 1, 1], + }, + "data_driven_decision_making_and_reasoning": { + "buckets": [ + EXPENSE_MANAGEMENT_TASKS, + MAXIMIZE_INVESTMENT_RETURN_TASKS, + DASH_COMPUTE_MEAN_AND_ORDER, + DASH_COMPUTE_MEDIAN_AND_ORDER, + DASH_COMPUTE_MODE_AND_ORDER, + DASH_COMPUTE_AND_CREATE_INCIDENT, + DASH_COMPUTE_AND_CREATE_PROBLEM, + DASH_COMPUTE_MEAN_FILTER_LIST, + DASH_COMPUTE_MEDIAN_FILTER_LIST, + DASH_COMPUTE_MODE_FILTER_LIST, + DASH_COMPUTE_MEAN_AND_REQUEST, + DASH_COMPUTE_MEDIAN_AND_REQUEST, + DASH_COMPUTE_MODE_AND_REQUEST, + ], + "num_seeds": 1, + "weights": [12, 28, 1, 1, 1, 3, 3, 1, 1, 1, 1, 1, 1], + }, + "sophisticated_memory": { + "buckets": [ + NAVIGATE_AND_CREATE_TASKS, + NAVIGATE_AND_ORDER_TASKS, + NAVIGATE_AND_FILTER_TASKS, + NAVIGATE_AND_SORT_TASKS, + OFFBOARD_USER_TASKS, + ONBOARD_USER_TASKS, + ], + "num_seeds": 8, + "weights": [1, 1, 1, 1, 1, 1], + }, + "contextual_understanding_infeasible_tasks": { + "buckets": [ + INFEASIBLE_NAVIGATE_AND_CREATE_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_CREATE, + INFEASIBLE_NAVIGATE_AND_ORDER_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_ORDER, + INFEASIBLE_NAVIGATE_AND_FILTER_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_FILTER, + INFEASIBLE_NAVIGATE_AND_SORT_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_SORT, + ], + "num_seeds": 4, + "weights": [1, 1, 1, 1, 1, 1, 1, 1], + }, +} + +HUMAN_CURRICULUM = { + "planning_and_problem_solving": { + "buckets": [ + MARK_DUPLICATE_PROBLEMS_TASKS, + WORKLOAD_BALANCING_TASKS, + WORK_ASSIGNMENT_TASKS, + SMALL_BASE_SCHEDULING_TASKS, + SMALL_TIGHT_SCHEDULING_TASKS, + ], + "num_seeds": 1, + "weights": [ + 3, + 1, + 2, + 1, + 1, + ], + }, + "information_retrieval": { + "buckets": [ + DASH_AND_ORDER, + DASH_AND_CREATE_INCIDENT, + DASH_AND_CREATE_PROBLEM, + DASH_COMPUTE_MIN_FILTER_LIST, + DASH_COMPUTE_MAX_FILTER_LIST, + DASH_AND_REQUEST, + WARRANTY_CHECK_TASKS, + FIND_AND_ORDER_ITEM_TASKS, + ], + "num_seeds": 1, + "weights": [1, 1, 1, 1, 1, 1, 1, 1], + }, + "data_driven_decision_making_and_reasoning": { + "buckets": [ + EXPENSE_MANAGEMENT_TASKS, + MAXIMIZE_INVESTMENT_RETURN_TASKS, # Not splitting as small multiplier + [ + *DASH_COMPUTE_MEAN_AND_ORDER, + *DASH_COMPUTE_MEDIAN_AND_ORDER, + *DASH_COMPUTE_MODE_AND_ORDER, + ], + [ + *DASH_COMPUTE_AND_CREATE_INCIDENT, + *DASH_COMPUTE_AND_CREATE_PROBLEM, + *DASH_COMPUTE_MEAN_AND_REQUEST, + ], + DASH_COMPUTE_MEAN_FILTER_LIST, + [ + *DASH_COMPUTE_MEDIAN_FILTER_LIST, + *DASH_COMPUTE_MODE_FILTER_LIST, + ], + [ + *DASH_COMPUTE_MEDIAN_AND_REQUEST, + *DASH_COMPUTE_MODE_AND_REQUEST, + ], + ], + "num_seeds": 1, + "weights": [2, 6, 1, 1, 1, 1, 1], + }, + "sophisticated_memory": { + "buckets": [ + NAVIGATE_AND_CREATE_TASKS, + NAVIGATE_AND_ORDER_TASKS, + NAVIGATE_AND_FILTER_TASKS, + NAVIGATE_AND_SORT_TASKS, + OFFBOARD_USER_TASKS, + ONBOARD_USER_TASKS, + ], + "num_seeds": 2, + "weights": [1, 1, 1, 1, 1, 1], + }, + "contextual_understanding_infeasible_tasks": { + "buckets": [ + INFEASIBLE_NAVIGATE_AND_CREATE_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_CREATE, + INFEASIBLE_NAVIGATE_AND_ORDER_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_ORDER, + INFEASIBLE_NAVIGATE_AND_FILTER_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_FILTER, + INFEASIBLE_NAVIGATE_AND_SORT_WITH_REASON, + INFEASIBLE_NAVIGATE_AND_SORT, + ], + "num_seeds": 1, + "weights": [1, 1, 1, 1, 1, 1, 1, 1], + }, +} diff --git a/src/browsergym/workarena/tasks/compositional/utils/infeasible_configs.py b/src/browsergym/workarena/tasks/compositional/utils/infeasible_configs.py new file mode 100644 index 0000000..a2751f2 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/utils/infeasible_configs.py @@ -0,0 +1,151 @@ +import numpy as np + +from faker import Faker + +fake = Faker() + + +def get_infeasible_form_config(config, random: np.random, provide_reason: bool = True): + """ + Get an infeasible form config from a feasible config by replacing the name of one of the task_fields with a random word + + Args: + -------- + config (dict): + The feasible form config to be transformed into an infeasible one + random (np.random): + The random number generator to use + provide_reason (bool): + Whether to provide a reason for the infeasibility. If False, the list of reasons will be [""] so that + any infeasibility can be detected by the absence of a reason + + Returns: + -------- + infeasible_config (dict): + The infeasible form config + infeasible_keywords (list[str]): + The name of the new field printed and its system name + """ + replaced_field = ( + random.choice(config["infeasible_task_fields"]) + if "infeasible_task_fields" in config + else random.choice(config["task_fields"]) + ) + new_field_printed = fake.word().capitalize() + " " + fake.word() + new_field_system_name = new_field_printed.lower().replace(" ", "_") + + config["task_fields"].remove(replaced_field) + config["task_fields"].append(new_field_system_name) + config["fields"][new_field_system_name] = new_field_printed + config["template_record"][new_field_system_name] = fake.word() + + infeasible_reasons = [new_field_printed, new_field_system_name] if provide_reason else [""] + + return config, infeasible_reasons + + +def get_infeasible_service_catalog_config(config, random: np.random, provide_reason: bool = True): + """ + Get an infeasible service catalog config from a feasible config by replacing the name of one of the additional configuration items with a random word + + Args: + -------- + config (dict): + The feasible service catalog config to be transformed into an infeasible one + random (np.random): + The random number generator to use + provide_reason (bool): + Whether to provide a reason for the infeasibility. If False, the list of reasons will be [""] so that + any infeasibility can be detected by the absence of a reason + + Returns: + -------- + infeasible_config (dict): + The infeasible service catalog config + infeasible_keywords (list[str]): + The name of the new field printed and its system name + """ + item_configuration = list(config["configuration"].keys()) + # if there is a configuration item, replace it with a new one; otherwise, simply add a new one + if item_configuration: + replaced_field = random.choice(item_configuration) + config["configuration"].pop(replaced_field) + new_field_printed = fake.word().capitalize() + " " + fake.word() + field_type = random.choice(["radio", "textarea", "checkbox", "select"]) + field_options = [fake.word() for _ in range(random.randint(2, 5))] + + config["configuration"][new_field_printed] = [field_type, ", ".join(field_options)] + + infeasible_reasons = [new_field_printed, *field_options] if provide_reason else [""] + + return config, infeasible_reasons + + +def get_infeasible_sort_config(config, random: np.random, provide_reason: bool = True): + """ + Get an infeasible sort config from a feasible config by replacing the name of one sort_fields with a random word + + Args: + -------- + config (dict): + The feasible sort config to be transformed into an infeasible one + random (np.random): + The random number generator to use + provide_reason (bool): + Whether to provide a reason for the infeasibility. If False, the list of reasons will be [""] so that + any infeasibility can be detected by the absence of a reason + + Returns: + -------- + infeasible_config (dict): + The infeasible sort config + infeasible_keywords (list[str]): + The name of the new sort option printed and its system name + """ + goal = config["goal"] + config_fields = [line[3:].split(" (")[0] for line in goal.split("\n")[1:]] + replaced_field_index = random.randint(0, len(config["sort_fields"])) + + new_field_printed = fake.word().capitalize() + " " + fake.word() + new_field_system_name = new_field_printed.lower().replace(" ", "_") + + config["goal"] = goal.replace(config_fields[replaced_field_index], new_field_printed) + config["sort_fields"][replaced_field_index] = new_field_system_name + + infeasible_reasons = [new_field_printed, new_field_system_name] if provide_reason else [""] + + return config, infeasible_reasons + + +def get_infeasible_filter_config(config, random: np.random, provide_reason: bool = True): + """ + Get an infeasible filter config from a feasible config by replacing the name of one of the filter_columns with a random word + + Args: + -------- + config (dict): + The feasible filter config to be transformed into an infeasible one + random (np.random): + The random number generator to use + provide_reason (bool): + Whether to provide a reason for the infeasibility. If False, the list of reasons will be [""] so that + any infeasibility can be detected by the absence of a reason + + Returns: + -------- + infeasible_config (dict): + The infeasible filter config + infeasible_keywords (list[str]): + The name of the new filter option printed and its system name + """ + replaced_field_index = random.randint(0, len(config["filter_columns"])) + + new_field_printed = fake.word().capitalize() + " " + fake.word() + new_field_system_name = new_field_printed.lower().replace(" ", "_") + config["filter_columns"][replaced_field_index] = new_field_system_name + config["filter_values"][replaced_field_index] = fake.word().capitalize() + config["list_info"]["columns"][new_field_system_name] = {"label": new_field_printed} + + infeasible_reasons = [new_field_printed, new_field_system_name] if provide_reason else [""] + + return config, infeasible_reasons diff --git a/src/browsergym/workarena/tasks/compositional/utils/knapsack.py b/src/browsergym/workarena/tasks/compositional/utils/knapsack.py new file mode 100644 index 0000000..85bd891 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/utils/knapsack.py @@ -0,0 +1,192 @@ +import numpy as np + + +class KnapsackInstanceGenarator: + """ + Generates a knapsack instance with the given number of items and maximum capacity, and solves it. + The instance is guaranteed to have a unique optimal solution in "random" or "single_item" mode . + + Args: + - random: Random number generator + - num_items: Number of items + - max_capacity: Maximum capacity of the knapsack + - mode: Mode of generation. Choice of "random", "trivial", "single_item", "single_item_uniform", "n_items" + - random: Randomly generate the instance and return it; guaranteed to have a unique optimal solution + - trivial: Generate a trivial instance with all items fitting in the knapsack; return the instance + - single_item: Generate an instance where the optimal solution has only one item + - n_items: Generate an instance with all items having uniform weight and value; n items fitting in the knapsack + - single_item_uniform: Generate an instance with all items having uniform weight and value; optimal solution has only one item and it can be any + - num_items_in_solution: Number of items in the optimal solution. Required for "n_items" mode. + - default_return: Default return value for investments having uniform weight and value. Required for "n_items" and "single_item_uniform" modes. + """ + + def __init__( + self, + random: np.random, + num_items: int, + max_capacity: int, + mode: str = "random", + num_items_in_solution: int = None, + default_return: int = 100000, + ): + self.random = random + self.num_items = num_items + self.max_capacity = max_capacity + self.mode = mode + self.num_items_in_solution = num_items_in_solution + self.default_return = default_return + + def get_instance(self): + if self.mode in ["random", "trivial"]: + return self.generate_and_solve_knapsack_instance() + elif self.mode == "single_item": + return self.generate_single_item_knapsack_instance() + elif self.mode in ["single_item_uniform", "n_items"]: + return self.generate_uniform_knapsack_instance() + else: + raise ValueError(f"Invalid mode {self.mode} for knapsack instance generation") + + def generate_and_solve_knapsack_instance(self): + """ + Generates a knapsack instance with the given number of items and maximum capacity, and solves it. + Used to generate instances for the "random" and "trivial" mode. + Returns: + - investments: List of tuples (cost, investment_return) for each investment + - max_return: Maximum return achievable with optimal solution + - selected_indices: Indices of the investments selected in the optimal solution + """ + + assert self.mode in [ + "random", + "trivial", + ], f"Mode {self.mode} is invalid for instance generation with generate_and_solve_knapsack_instance" + + multiple_solutions = True + while multiple_solutions: + # Generate knapsack instance... + investments = [] + min_cost = self.max_capacity // (self.num_items * 2) + max_cost = ( + self.max_capacity // 2 + if self.mode == "random" + else self.max_capacity // self.num_items + ) + for _ in range(self.num_items): + cost = self.random.randint(min_cost, max_cost) + # Ensure that investments yield positive returns + investment_return = self.random.randint( + self.max_capacity // 2, self.max_capacity // 2 + 40000 + ) + investments.append((cost, investment_return)) + + total_cost = sum([investments[i][0] for i in range(self.num_items)]) + # Skip trivial instances where all items fit in the knapsack + if self.mode == "random" and total_cost <= self.max_capacity: + continue + + if self.mode == "random": + # ...Solve it... + max_return, num_optimal_solutions, selected_indices = self.solve_knapsack( + investments, self.max_capacity + ) + # ...and check if there are multiple solutions + multiple_solutions = num_optimal_solutions > 1 + else: + selected_indices = list(range(self.num_items)) + max_return = sum([investments[i][1] for i in selected_indices]) + multiple_solutions = False + + return investments, max_return, selected_indices + + def generate_single_item_knapsack_instance(self): + """Generate knapsack instance where the optimal solution contains only one item + Returns: + - investments: List of tuples (cost, investment_return) for each investment + - max_investment_return: Investment return of the selected investment in the optimal solution + - selected_indices: Index of the selected investment in the optimal solution + """ + assert ( + self.mode == "single_item" + ), f"Mode {self.mode} is invalid for instance generation with generate_single_item_knapsack_instance" + + # Ensure that the optimal solution contains only one item + min_cost = self.max_capacity // 2 + 1 + max_cost = self.max_capacity - 1 + + max_investment_return = 0 + max_investment_index = 0 + + # Generate knapsack instance... + investments = [] + for i in range(self.num_items): + cost = self.random.randint(min_cost, max_cost) + investment_return = self.random.randint(max_cost, 2 * max_cost) + + # Ensure that the optimal solution contains only one item + while investment_return == max_investment_return: + investment_return = self.random.randint(max_cost, 2 * max_cost) + + if investment_return > max_investment_return: + max_investment_return = investment_return + max_investment_index = i + + investments.append((cost, investment_return)) + + return investments, max_investment_return, [max_investment_index] + + def generate_uniform_knapsack_instance(self): + """Generate knapsack instance where all items have the same cost and return + Returns: + - investments: List of tuples (cost, investment_return) for each investment + - max_return: Maximum return achievable with optimal solution + - selected_indices=None: No need to return selected indices as all items have the same cost and return. The validation code should check that + the optimal solution contains a subset of the items of the right length. + """ + assert self.mode in [ + "single_item_uniform", + "n_items", + ], f"Mode {self.mode} is invalid for instance generation with generate_n_items_knapsack_instance" + items_in_solution = self.num_items_in_solution if self.mode == "n_items" else 1 + + # Ensure that the optimal solution contains the specified number of items + item_weight = self.max_capacity // (items_in_solution + 1) + 1 + # Generate knapsack instance... + investments = [(item_weight, self.default_return) for _ in range(self.num_items)] + + return investments, self.default_return * items_in_solution, None + + def solve_knapsack(self, investments, max_capacity): + """Solves the knapsack problem using dynamic programming""" + num_investments = len(investments) + + # Initialize DP table for maximum return and number of ways + dp = [[(0, 0) for _ in range(max_capacity + 1)] for _ in range(num_investments + 1)] + + for i in range(1, num_investments + 1): + cost, return_ = investments[i - 1] + for w in range(max_capacity + 1): + if cost <= w: + # If adding the current investment yields a higher return, update the cell + if return_ + dp[i - 1][w - cost][0] > dp[i - 1][w][0]: + dp[i][w] = (return_ + dp[i - 1][w - cost][0], 1) + # If it yields the same return, add the number of ways from the cell without the current investment + elif return_ + dp[i - 1][w - cost][0] == dp[i - 1][w][0]: + dp[i][w] = (dp[i - 1][w][0], dp[i - 1][w][1] + dp[i - 1][w - cost][1]) + # If it yields a lower return, keep the old maximum return and number of ways + else: + dp[i][w] = dp[i - 1][w] + else: + dp[i][w] = dp[i - 1][w] + + # Retrieve the maximum return and the number of ways to achieve it + max_return, num_ways = dp[num_investments][max_capacity] + + # Retrieve the indices of the selected investments + selected_indices = [] + w = max_capacity + for i in range(num_investments, 0, -1): + if dp[i][w] != dp[i - 1][w]: + selected_indices.append(i - 1) + w -= investments[i - 1][0] + + return max_return, num_ways, selected_indices diff --git a/src/browsergym/workarena/tasks/compositional/warranty_check.py b/src/browsergym/workarena/tasks/compositional/warranty_check.py new file mode 100644 index 0000000..70c4dc1 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/warranty_check.py @@ -0,0 +1,227 @@ +import json +import time +from faker import Faker + +fake = Faker() + +from playwright.sync_api._generated import Page + +from .base import CompositionalTask, HumanEvalTask + +from ..base import AbstractServiceNowTask +from ..knowledge import KnowledgeBaseSearchTask +from ..list import ExtractListInfoTask, FilterHardwareListTask +from ..navigation import AllMenuTask + +from ...api.computer_asset import create_computer_asset +from ...api.user import create_user +from ...api.utils import db_delete_from_table, table_api_call +from ...instance import SNowInstance + + +class GetWarrantyExpirationDateTask(CompositionalTask, HumanEvalTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Create a compositional task with specific subtasks + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of subtasks + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + Attributes: + ----------- + task_description: str + The start of the task description to be completed. e.g. "Referring to company protocol '[company_protocol]', onboard user with the following information: \n" + short_description: str + A short description of the task to be completed. e.g. "Find the warranty expiration date for John Doe's laptop" + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + self.protocol_name = "Finding the warranty expiration for a user's laptop" + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + protocol_name=self.protocol_name, + ) + self.task_description = None + self.short_description = None + self.user_sys_id = None + self.user_name = None + self.user_full_name = None + self.laptop_sys_id = None + self.warranty_expiration_date = None + + def setup_goal(self, page: Page) -> tuple[str, dict]: + # Create a user and a laptop for the user + self._create_user_and_laptop() + + # Sample a configuration + config = self.fixed_config if self.fixed_config else self._get_config() + + # Get the task description + self.short_description = ( + f"Find the warranty expiration date for {self.user_full_name}'s laptop" + ) + self.task_description = f'Refer to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) for the steps to find the Warranty expiration date for {self.user_full_name}\'s laptop.\n \n' + + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[AbstractServiceNowTask]: + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": f'Can you find the "{self.protocol_name}" Protocol in the Knowledge Base?', + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + navigate_to_hardware_asset_and_filter = [ + # Navigate to the hardware asset list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Asset", + "module": "Portfolios > Hardware Assets", + "url": "/now/nav/ui/classic/params/target/alm_hardware_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter the hardware asset list + FilterHardwareListTask( + instance=self.instance, + fixed_config={ + "filter_columns": ["assigned_to"], + "filter_kind": "AND", + "filter_values": [f"{self.user_full_name}"], + }, + is_validated=False, + used_in_level_2=True, + ), + ] + extract_warranty_subtask = [ + ExtractListInfoTask( + instance=self.instance, + unique_field_name="assigned_to", + fixed_config={ + "start_rel_url": "", + "fields": { + "assigned_to": "Assigned to", + "warranty_expiration": "Warranty expiration", + }, + "expected_values": [ + { + "assigned_to": f"{self.user_full_name}", + "warranty_expiration": f"{self.warranty_expiration_date}", + } + ], + }, + list_name="Hardware Assets", + list_url="/now/nav/ui/classic/params/target/alm_hardware_list.do", + table_name="alm_hardware", + is_validated=True, + used_in_level_2=True, + ), + ] + + config = ( + navigate_to_protocol_subtask + + navigate_to_hardware_asset_and_filter + + extract_warranty_subtask + ) + + return config + + def _create_user_and_laptop(self) -> None: + """ + Creates a user and a laptop for the user. The laptop model is randomly selected from the available computer models. + Sets the user_sys_id, laptop_sys_id, user_name and warranty_expiration_date attributes. + """ + # Generate random name for the user + first_name = fake.first_name() + "-" + fake.first_name() + last_name = fake.last_name() + "-" + fake.last_name() + self.user_full_name = first_name + " " + last_name + # Create user + self.user_name, _, self.user_sys_id = create_user( + instance=self.instance, first_name=first_name, last_name=last_name, random=self.random + ) + + assert self.user_sys_id, f"Failed to create user {first_name} {last_name}" + self.warranty_expiration_date = str(fake.date_between(start_date="-1y", end_date="+1y")) + asset_tag = "P" + str(id(self) % (10**8)).zfill(8) + ( + computer_sys_id, + _, + _, + ) = create_computer_asset( + instance=self.instance, + asset_tag=asset_tag, + warranty_expiration_date=self.warranty_expiration_date, + user_sys_id=self.user_sys_id, + random=self.random, + ) + + assert computer_sys_id, f"Failed to create hardware asset {asset_tag}" + self.laptop_sys_id = computer_sys_id + + def teardown(self) -> None: + # Delete the user and the laptop + user_record_exists = table_api_call( + instance=self.instance, + table="sys_user", + params={"sysparm_query": f"sys_id={self.user_sys_id}"}, + ) + laptop_record_exists = table_api_call( + instance=self.instance, + table="alm_hardware", + params={"sysparm_query": f"sys_id={self.laptop_sys_id}"}, + ) + if user_record_exists: + db_delete_from_table( + instance=self.instance, + table="sys_user", + sys_id=self.user_sys_id, + ) + if laptop_record_exists: + db_delete_from_table( + instance=self.instance, + table="alm_hardware", + sys_id=self.laptop_sys_id, + ) + super().teardown() + + +__TASKS__ = [GetWarrantyExpirationDateTask] diff --git a/src/browsergym/workarena/tasks/compositional/work_assignment.py b/src/browsergym/workarena/tasks/compositional/work_assignment.py new file mode 100644 index 0000000..90e8ad0 --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/work_assignment.py @@ -0,0 +1,804 @@ +from typing import Tuple +from faker import Faker +import random + +fake = Faker() + +from playwright.sync_api._generated import Page + +from .base import CompositionalTask, HumanEvalTask + +from ...api.incident import create_incident +from ...api.user import create_user +from ...api.utils import table_api_call, db_delete_from_table +from ..base import AbstractServiceNowTask +from ..list import FilterIncidentListTask +from ..form import EditIncidentTask +from ..knowledge import KnowledgeBaseSearchTask +from ..navigation import AllMenuTask + +from ...instance import SNowInstance + + +class WorkAssignmentTask(CompositionalTask): + def __init__( + self, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_experts_per_category: int = 2, + max_assignments: int = None, + min_assignments: int = None, + num_categories: int = None, + seed: int = None, + prefix: str = None, + ) -> None: + """ + Create a compositional task with specific subtasks + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[tuple[AbstractServiceNowTask, dict, bool]] + A list of tuples, each containing a subtask, its configuration and whether or not it should be validated. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + max_experts_per_category: int + How many maximum new agents to create for each category. + max_assignments: int + Maximum number of incidents created to be assigned. + For a task, the number is randomly sampled between max_assignments and min_assignments. + max_assignments: int + Minimum number of incidents created to be assigned. + For a task, the number is randomly sampled between max_assignments and min_assignments. + prefix: str + Prefix to name the incidents created with a unique prefix + Attributes: + ----------- + task_description: str + The start of the task description to be completed. e.g. "Referring to company protocol 'Work Assignment', assign incidents to different agents with the following information: \n" + short_description: str + A short description of the task to be completed. e.g. "Assign task to relevant expert agents" + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + self.protocol_name = "Work Assignment: Assign Incidents to Relevant Agents" + super().__init__( + instance=instance, + fixed_config=fixed_config, + level=level, + protocol_name=self.protocol_name, + seed=seed, + ) + + self.task_description = None + self.short_description = f"Assign work to relevant agents" + self.max_experts_per_category = max_experts_per_category + self.max_assignments = max_assignments + self.min_assignments = min_assignments + self.num_categories = num_categories + if self.num_categories > 4 or self.num_categories < 1: + raise Exception("Should have at least 1 and at most 4 categories.") + self.prefix = prefix + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.incident_configs = [] + number_assignments = self.random.randint(self.min_assignments, self.max_assignments) + + all_existing_incidents = table_api_call( + instance=self.instance, table="incident", method="GET" + )["result"] + all_incident_numbers = [incident["number"] for incident in all_existing_incidents] + new_incident_numbers = [] + for _ in range(number_assignments): + incident_number = ( + self.prefix + str(id(self) % (10**8)).zfill(8)[:4] + str(random.randint(100, 999)) + ) + while ( + incident_number in all_incident_numbers or incident_number in new_incident_numbers + ): + incident_number = ( + self.prefix + + str(id(self) % (10**8)).zfill(8)[:4] + + str(random.randint(100, 999)) + ) + new_incident_numbers.append(incident_number) + + self.active_categories = self.random.choice( + ["hardware", "software", "network", "database"], self.num_categories, replace=False + ) + for incident_number in new_incident_numbers: + ### We can reduce the categories here if the setup takes too long + category = self.random.choice(self.active_categories) + incident_response = create_incident( + instance=self.instance, + incident_number=incident_number, + caller_sys_id=self._base_user_sysid, + category=category, + priority=4, + impact=2, # priority is calculated as some combination of impact and urgency + urgency=3, + ) + self.incident_configs.append(incident_response) + + self.experts = dict({category: [] for category in self.active_categories}) + for _ in range(self.max_experts_per_category): + for category in self.active_categories: + self.experts[category].append( + create_user( + instance=self.instance, + first_name=f"{fake.first_name()}-{fake.first_name()}", + last_name=f"{fake.last_name()}-{fake.last_name()}", + return_full_response=True, + user_roles=["itil"], + random=self.random, + ) + ) + expert_string = "" + for category in self.active_categories: + category_experts = ", ".join( + expert["first_name"] + " " + expert["last_name"] + for expert in self.experts[category] + ) + expert_string += f"{category.capitalize()} agents: {category_experts} \n" + incident_numbers = ", ".join(new_incident_numbers) + + # Get the task description + self.task_description = ( + f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) assign work to the agents with the following information: \n' + + f"Incidents to assign: {incident_numbers} \n\n" + + f"{expert_string}" + ) + + # Sample a configuration + config = self.fixed_config if self.fixed_config else self._get_config() + + goal, info = super().setup_goal(page=page, config=config) + + if self.level == 2: + goal = ( + self.short_description + + f"\n1. Navigate to the Service Desk > Incidents. \n" + + f"\n2. You have to assign the following incidents to relevant agents: {incident_numbers}. You can filter the list using each incident number and use the 'Assigned to' field to assign an incident.\n" + + f"\n3. You have to ensure that each incident is assigned to a relevant agent based on the category of the incident.\n" + + f"\nThe category wise agents are as follows. You can assign an incident to ANY agent from the category:\n" + + f"{expert_string}" + ) + + return goal, info + + def _get_config(self) -> list[tuple[AbstractServiceNowTask, dict, bool]]: + + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": "Can you find the Work Assignment Protocol in the Knowledge Base?", + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + + all_incident_assignments = [] + + for incident_config in self.incident_configs: + assigned_to = self.random.choice(self.experts[incident_config["category"]]) + assigned_to = assigned_to["first_name"] + " " + assigned_to["last_name"] + assign_incidents_subtask = [ + # Navigate to the incidents list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Service Desk", + "module": "Incidents", + "url": "/now/nav/ui/classic/params/target/incident_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter incident + FilterIncidentListTask( + instance=self.instance, + fixed_config={ + "filter_columns": [ + "number", + ], + "filter_kind": "AND", + "filter_values": [ + incident_config["number"], + ], + }, + is_validated=False, + used_in_level_2=True, + ), + # Edit incident + EditIncidentTask( + instance=self.instance, + # fixed_config=incident_config, + new_values={"assigned_to": assigned_to}, + is_validated=False, + used_in_level_2=True, + record_sys_id=incident_config["sys_id"], + level=self.level, + ), + ] + all_incident_assignments.extend(assign_incidents_subtask) + + config = navigate_to_protocol_subtask + all_incident_assignments + + return config + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + experts_sys_ids = { + category: [expert["sys_id"] for expert in self.experts[category]] + for category in self.experts + } + for incident_config in self.incident_configs: + incident_response = table_api_call( + instance=self.instance, + table="incident", + params={ + "sysparm_query": f"sys_id={incident_config['sys_id']}", + "sysparm_fields": "category,assigned_to", + }, + method="GET", + )["result"][0] + if incident_response["category"] != incident_config["category"]: + raise Exception("Corrupted incident data") + if not incident_response["assigned_to"]: + return ( + 0, + False, + "", + { + "message": f"The incident {incident_config['number']} has not been assigned to anyone." + }, + ) + if ( + incident_response["assigned_to"]["value"] + not in experts_sys_ids[incident_response["category"]] + ): + return ( + 0, + False, + "", + { + "message": f"The incident {incident_config['number']} was assigned to an incorrect expert." + }, + ) + # Validate final_l3 tasks + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + + def teardown(self) -> None: + for incident in self.incident_configs: + db_delete_from_table( + instance=self.instance, table="incident", sys_id=incident["sys_id"] + ) + + for experts in self.experts.values(): + for expert in experts: + db_delete_from_table( + instance=self.instance, table="sys_user", sys_id=expert["sys_id"] + ) + + return super().teardown() + + +class WorkAssignmentSmallTask(WorkAssignmentTask, HumanEvalTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Small version of workassignment task. + """ + super().__init__( + instance=instance, + level=level, + max_experts_per_category=2, + max_assignments=4, + min_assignments=3, + num_categories=2, + fixed_config=fixed_config, + seed=seed, + prefix="WAS", + ) + + +class WorkAssignmentMediumTask(WorkAssignmentTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Medium version of workassignment task. + """ + super().__init__( + instance=instance, + level=level, + max_experts_per_category=2, + max_assignments=6, + min_assignments=5, + num_categories=3, + fixed_config=fixed_config, + seed=seed, + prefix="WAM", + ) + + +class WorkAssignmentLargeTask(WorkAssignmentTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + ) -> None: + """ + Large version of workassignment task. + """ + super().__init__( + instance=instance, + level=level, + max_experts_per_category=2, + max_assignments=8, + min_assignments=7, + num_categories=4, + fixed_config=fixed_config, + seed=seed, + prefix="WAL", + ) + + +class PriorityAssignmentTask(CompositionalTask): + def __init__( + self, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + max_tasks_per_priority: int = 2, + min_tasks_per_priority: int = 1, + num_categories: int = None, + seed: int = None, + prefix: str = None, + ) -> None: + """ + Create a compositional task with specific subtasks + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[tuple[AbstractServiceNowTask, dict, bool]] + A list of tuples, each containing a subtask, its configuration and whether or not it should be validated. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + experts_per_category: int + How many new agents to create for each category. + max_assignments: int + Maximum number of incidents created to be assigned. + For a task, the number is randomly sampled between max_assignments and min_assignments. + max_assignments: int + Minimum number of incidents created to be assigned. + For a task, the number is randomly sampled between max_assignments and min_assignments. + prefix: str + Prefix to name the incidents created with a unique prefix + Attributes: + ----------- + task_description: str + The start of the task description to be completed. e.g. "Referring to company protocol 'Priority Assignment', assign incidents to different agents in terms of priority with the following information: \n" + short_description: str + A short description of the task to be completed. e.g. "Assign task to relevant expert agents based on the incident priorities" + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + self.protocol_name = "Work Assignment: Assign Incidents to Relevant Agents" + super().__init__( + instance=instance, + fixed_config=fixed_config, + level=level, + protocol_name=self.protocol_name, + seed=seed, + ) + + self.task_description = None + self.short_description = None + self.experts_per_category = 3 # We divide agents into 'expert', 'supporter', and 'planner' + self.max_tasks_per_priority = max_tasks_per_priority + self.min_tasks_per_priority = min_tasks_per_priority + # Priority 1 is urgent, 3 is moderate, 5 is planning. + # Also priority depends on impact and urgency rather than being an independent attribute + self.priorities = { + 1: { + "impact": 1, + "urgency": 1, + "num_incidents": self.random.randint( + self.min_tasks_per_priority, self.max_tasks_per_priority + ), + "agent_type": "expert", + }, + 3: { + "impact": 2, + "urgency": 2, + "num_incidents": self.random.randint( + self.min_tasks_per_priority, self.max_tasks_per_priority + ), + "agent_type": "supporter", + }, + 5: { + "impact": 3, + "urgency": 3, + "num_incidents": self.random.randint( + self.min_tasks_per_priority, self.max_tasks_per_priority + ), + "agent_type": "planner", + }, + } + self.num_categories = num_categories + if self.num_categories > 4 or self.num_categories < 1: + raise Exception("Should have at least 1 and at most 4 categories.") + self.prefix = prefix + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.incident_configs = [] + number_assignments = sum( + [attribute["num_incidents"] for attribute in self.priorities.values()] + ) + + all_existing_incidents = table_api_call( + instance=self.instance, table="incident", method="GET" + )["result"] + all_incident_numbers = [incident["number"] for incident in all_existing_incidents] + + new_incident_numbers = [] + for _ in range(number_assignments): + incident_number = ( + self.prefix + str(id(self) % (10**8)).zfill(8)[:4] + str(random.randint(100, 999)) + ) + while ( + incident_number in all_incident_numbers or incident_number in new_incident_numbers + ): + incident_number = ( + self.prefix + + str(id(self) % (10**8)).zfill(8)[:4] + + str(random.randint(100, 999)) + ) + new_incident_numbers.append(incident_number) + incident_category = [] + self.active_categories = self.random.choice( + ["hardware", "software", "network", "database"], self.num_categories, replace=False + ) + incident_number_idx = 0 + for priority, attributes in self.priorities.items(): + for _ in range(attributes["num_incidents"]): + category = self.random.choice(self.active_categories) + incident_response = create_incident( + instance=self.instance, + incident_number=new_incident_numbers[incident_number_idx], + caller_sys_id=self._base_user_sysid, + category=category, + priority=priority, + impact=attributes[ + "impact" + ], # priority is calculated as some combination of impact and urgency + urgency=attributes["urgency"], + ) + self.incident_configs.append(incident_response) + incident_category.append( + [new_incident_numbers[incident_number_idx], category, priority] + ) + incident_number_idx += 1 + + self.agents_per_category = dict({category: {} for category in self.active_categories}) + for category in self.agents_per_category: + self.agents_per_category[category]["expert"] = create_user( + instance=self.instance, + first_name=f"{fake.first_name()}-{fake.first_name()}", + last_name=f"{fake.last_name()}-{fake.last_name()}", + return_full_response=True, + user_roles=["itil"], + random=self.random, + ) + self.agents_per_category[category]["expert"]["full_name"] = ( + self.agents_per_category[category]["expert"]["first_name"] + + " " + + self.agents_per_category[category]["expert"]["last_name"] + ) + + self.agents_per_category[category]["supporter"] = create_user( + instance=self.instance, + first_name=f"{fake.first_name()}-{fake.first_name()}", + last_name=f"{fake.last_name()}-{fake.last_name()}", + return_full_response=True, + user_roles=["itil"], + random=self.random, + ) + self.agents_per_category[category]["supporter"]["full_name"] = ( + self.agents_per_category[category]["supporter"]["first_name"] + + " " + + self.agents_per_category[category]["supporter"]["last_name"] + ) + + self.agents_per_category[category]["planner"] = create_user( + instance=self.instance, + first_name=f"{fake.first_name()}-{fake.first_name()}", + last_name=f"{fake.last_name()}-{fake.last_name()}", + return_full_response=True, + user_roles=["itil"], + random=self.random, + ) + self.agents_per_category[category]["planner"]["full_name"] = ( + self.agents_per_category[category]["planner"]["first_name"] + + " " + + self.agents_per_category[category]["planner"]["last_name"] + ) + + incident_numbers = ", ".join(new_incident_numbers) + + expert_string = "" + for category in self.active_categories: + category_experts = f"Expert: {self.agents_per_category[category]['expert']['full_name']}, Supporter: {self.agents_per_category[category]['supporter']['full_name']}, Planner: {self.agents_per_category[category]['planner']['full_name']}" + expert_string += f"{category.capitalize()} agents - {category_experts} \n" + # Get the task description + self.short_description = f"Assign work using priority to relevant agents" + self.task_description = ( + f'Referring to company protocol "{self.protocol_name}" (located in the "Company Protocols" knowledge base) assign work to the agents with the following information: \n' + + f"Incidents to assign: {incident_numbers} \n\n" + + f"{expert_string}" + ) + # Sample a configuration + config = self.fixed_config if self.fixed_config else self._get_config() + + goal, info = super().setup_goal(page=page, config=config) + + if self.level == 2: + goal = ( + self.short_description + + f"\n1. Navigate to the Service Desk > Incidents. \n" + + f"\n2. You have to assign the following incidents to relevant agents: {incident_numbers}. You can filter the list using each incident number and use the 'Assigned to' field to assign an incident.\n" + + f"\n3. You have to ensure that each incident is assigned to a relevant agent based on the priority of the incident and its category. For an incident with priority 1 - Critical, assign it to an 'expert' agent of the category, for priority 3 - Moderate, assign it to a 'supporter' of the category, and for priority 5 - Planning assign it to a 'planner' of the category.\n" + + f"\nThe category wise relevant agent are as follows:\n" + + f"{expert_string}" + ) + + return goal, info + + def _get_config(self) -> list[tuple[AbstractServiceNowTask, dict, bool]]: + + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": "Can you find the Work Assignment Protocol in the Knowledge Base?", + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + + all_incident_assignments = [] + + for incident_config in self.incident_configs: + assigned_to = self.agents_per_category[incident_config["category"]][ + self.priorities[int(incident_config["priority"])]["agent_type"] + ]["full_name"] + assign_incidents_subtask = [ + # Navigate to the incidents list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Service Desk", + "module": "Incidents", + "url": "/now/nav/ui/classic/params/target/incident_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter incident + FilterIncidentListTask( + instance=self.instance, + fixed_config={ + "filter_columns": [ + "number", + ], + "filter_kind": "AND", + "filter_values": [ + incident_config["number"], + ], + }, + is_validated=False, + used_in_level_2=True, + ), + # Edit incident + EditIncidentTask( + instance=self.instance, + # fixed_config=incident_config, + new_values={"assigned_to": assigned_to}, + is_validated=False, + used_in_level_2=True, + record_sys_id=incident_config["sys_id"], + level=self.level, + ), + ] + all_incident_assignments.extend(assign_incidents_subtask) + + config = navigate_to_protocol_subtask + all_incident_assignments + + return config + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + agents_per_category_sys_ids = { + category: { + agent_type: agent["sys_id"] + for agent_type, agent in self.agents_per_category[category].items() + } + for category in self.agents_per_category + } + for incident_config in self.incident_configs: + incident_response = table_api_call( + instance=self.instance, + table="incident", + params={ + "sysparm_query": f"sys_id={incident_config['sys_id']}", + "sysparm_fields": "category,assigned_to,priority", + }, + method="GET", + )["result"][0] + if incident_response["category"] != incident_config["category"]: + raise Exception("Corrupted incident data") + if not incident_response["assigned_to"]: + return ( + 0, + False, + "", + { + "message": f"The incident {incident_config['number']} has not been assigned to anyone." + }, + ) + if ( + incident_response["assigned_to"]["value"] + != agents_per_category_sys_ids[incident_response["category"]][ + self.priorities[int(incident_response["priority"])]["agent_type"] + ] + ): + return ( + 0, + False, + "", + { + "message": f"The incident {incident_config['number']} was assigned to an incorrect agent." + }, + ) + # Validate final_l3 tasks + reward, done, message, info = super().validate(page, chat_messages) + return reward, done, message, info + + def teardown(self) -> None: + for incident in self.incident_configs: + db_delete_from_table( + instance=self.instance, table="incident", sys_id=incident["sys_id"] + ) + + for category in self.agents_per_category.values(): + for agent in category.values(): + db_delete_from_table( + instance=self.instance, table="sys_user", sys_id=agent["sys_id"] + ) + + return super().teardown() + + +class PriorityAssignmentSmallTask(PriorityAssignmentTask, HumanEvalTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 3, + ) -> None: + """ + Small version of priority assignment task. + """ + super().__init__( + instance=instance, + level=level, + num_categories=2, + fixed_config=fixed_config, + seed=0, + prefix="PAS", + ) + + +class PriorityAssignmentMediumTask(PriorityAssignmentTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 3, + ) -> None: + """ + Medium version of priority assignment task. + """ + super().__init__( + instance=instance, + level=level, + num_categories=3, + fixed_config=fixed_config, + seed=seed, + prefix="PAM", + ) + + +class PriorityAssignmentLargeTask(PriorityAssignmentTask): + def __init__( + self, + instance: SNowInstance = None, + seed: int = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 3, + ) -> None: + """ + Large version of priority assignment task. + """ + super().__init__( + instance=instance, + level=level, + num_categories=4, + fixed_config=fixed_config, + seed=seed, + prefix="PAL", + ) + + +__TASKS__ = [ + WorkAssignmentSmallTask, + WorkAssignmentMediumTask, + WorkAssignmentLargeTask, + PriorityAssignmentSmallTask, + PriorityAssignmentMediumTask, + PriorityAssignmentLargeTask, +] diff --git a/src/browsergym/workarena/tasks/compositional/workload_balancing.py b/src/browsergym/workarena/tasks/compositional/workload_balancing.py new file mode 100644 index 0000000..d79b86b --- /dev/null +++ b/src/browsergym/workarena/tasks/compositional/workload_balancing.py @@ -0,0 +1,396 @@ +import faker + +fake = faker.Faker() + +from playwright.sync_api._generated import Page + +from .base import CompositionalTask, HumanEvalTask + +from ..base import AbstractServiceNowTask +from ..dashboard import WorkLoadBalancingMinMaxRetrievalTask +from ..form import EditProblemTask +from ..knowledge import KnowledgeBaseSearchTask +from ..list import FilterProblemListForWorkLoadBalancingTask +from ..navigation import AllMenuTask +from ..send_chat_message import SendChatMessageGenericTask + +from ...api.problem import create_problem +from ...api.report import create_report +from ...api.user import create_user +from ...api.utils import db_delete_from_table, table_api_call +from ...instance import SNowInstance + + +class WorkloadBalancingTask(CompositionalTask): + def __init__( + self, + seed: int = None, + instance: SNowInstance = None, + fixed_config: list[AbstractServiceNowTask] = None, + level: int = 2, + min_users: int = 2, + max_users: int = 4, + # Ranges to randomly choose from + max_problem_range: int = [3, 4], + mid_problem_range: int = [2, 3], + min_problem_range: int = [1, 2], + ) -> None: + """ + Workload balancing task: + - Navigate to the KB + - Find the protocol for re-distributing work + - Find the user who has the greatest number of problems assigned to them + - Re-assign the problems to the user having the least number of problems assigned to them + + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: list[AbstractServiceNowTask] + A list of subtasks. + level: int + The level of the task; choice between 2 and 3. L2 will have all the info in the the goal and start in the SNOW home page. + L3 will start in a private task page describing the information needed to complete the task and the related company protocol + to complete it. + min_users: int + The minimum number of users to create and to distribute the problems to + max_users: int + The maximum number of users to create and to distribute the problems to + max_problem_range: list[int, int] + The range of the number of problems to assign to the user with the most problems + mid_problem_range: list[int, int] + The range of the number of problems to assign to all users but the ones with the least/most problems + min_problem_range: list[int, int] + The range of the number of problems to assign to the user with the least problems + Attributes: + ----------- + task_description: str + The start of the task description to be completed. e.g. "Referring to company protocol 'Agent Workload Balancing', re-distribute the problems with description containing {self.problem_hashtag}" + short_description: str + A short description of the task to be completed. e.g. "Balance the workload for problems with description containing {self.problem_hashtag}" + """ + assert level in [2, 3], "Level must be either 2 or 3" + self.level = level + self.protocol_name = "Agent Workload Balancing" + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + level=level, + protocol_name=self.protocol_name, + ) + + self.problem_hashtag = ( + f"#PRB{str(id(self) % (10**8)).zfill(9)}" # identifier to select problems + ) + self.task_description = None + self.short_description = None + self.min_users = min_users + self.max_users = max_users + self.max_problem_range = max_problem_range + self.mid_problem_range = mid_problem_range + self.min_problem_range = min_problem_range + + # In this case, there will only be 2 users as the values are bound and the top value is excluded in the randint function + if self.min_users == 2 and self.max_users == 3: + assert ( + self.min_problem_range[1] <= self.max_problem_range[0] + ), "The problem ranges should not overlap" + # In this case, there will be 3 users + else: + assert ( + self.min_problem_range[1] <= self.mid_problem_range[0] + and self.mid_problem_range[1] <= self.max_problem_range[0] + ), "The problem ranges should not overlap" + assert self.max_problem_range[1] <= 6, "The maximum number of problems should not exceed 6" + + self.plot_title = None # The title of the plot created for the report + self.lowest_priority = 0 # The lowest priority of the problems; a high number indicates a low priority. Set in the setup_goal method + + self.category_name = ( + fake.word() + "-" + fake.word() + ) # The category of the problems to re-distribute + self.category_sys_id = ( + None # The sys_id of the category created for the task; create in the setup_goal method + ) + self.user_sys_ids = [] # The sys_ids of the users created for the task + self.problem_sys_ids = [] # The sys_ids of the problems created for the task + self.report_sys_id = None # The sys_id of the report created for the task + self.user_with_most_problems = None # The name of the user that has the most problems assigned; defined in the setup_goal method + self.user_with_least_problems = None # The name of the user that has the least problems assigned; defined in the setup_goal method + self.problem_to_edit_sys_id = ( + None # The sys_id of the problem to re-assign; defined in the setup_goal method + ) + self.problem_to_edit_number = ( + None # The number of the problem to re-assign; defined in the setup_goal method + ) + + def setup_goal(self, page: Page) -> tuple[str, dict]: + num_users = self.random.randint(self.min_users, self.max_users) + max_problems = self.random.randint(*self.max_problem_range) + min_problems = self.random.randint(*self.min_problem_range) + + # Create users, create problems and assign problems to users + for i in range(num_users): + if i == 0: + num_problems = max_problems + elif i == num_users - 1: + num_problems = min_problems + else: + num_problems = self.random.randint(*self.mid_problem_range) + first_name = fake.first_name() + "-" + fake.first_name() + last_name = fake.last_name() + "-" + fake.last_name() + user_full_name = first_name + " " + last_name + _, _, user_sys_id = create_user( + instance=self.instance, + first_name=first_name, + last_name=last_name, + user_roles=[ + "admin", + "problem_admin", + "problem_manager", + "problem_coordinator", + "problem_task_analyst", + ], + random=self.random, + ) + self.user_sys_ids.append(user_sys_id) + + if i == 0: + self.user_with_most_problems = user_full_name + elif i == num_users - 1: + self.user_with_least_problems = user_full_name + + # Create problems assigned to current user + for j in range(num_problems): + # Assign a priority to the problem; 1 being highest priority and 5 being lowest + # the use of j % 5 is to ensure that the priority is between 1 and 5 and that there is + # only one problem with the lowest priority + priority = (j % 5) + 1 + self.lowest_priority = max(self.lowest_priority, priority) + problem_sys_id, problem_number = create_problem( + instance=self.instance, + user_sys_id=user_sys_id, + priority=priority, + problem_hashtag=self.problem_hashtag, + return_number=True, + ) + # The last problem created is the one to re-assign as it will be the one with the lowest priority (highest priority value) + # and the first user will be the one with the most problems assigned + if i == 0 and j == num_problems - 1: + self.problem_to_edit_sys_id = problem_sys_id + self.problem_to_edit_number = problem_number + self.problem_sys_ids.append(problem_sys_id) + + # Create a report for problems of the current category + self.report_sys_id, plot_title = create_report( + instance=self.instance, + table="problem", + filter_hashtag=self.problem_hashtag, + field="assigned_to", + plot_title=f"Problems for with hashtag {self.problem_hashtag}", + random=self.random, + ) + self.plot_title = plot_title + + # Sample a configuration + config = self._get_config() + # Get the task description + self.short_description = ( + f"Balance the workload for problems with hashtag {self.problem_hashtag}" + ) + self.task_description = f"Referring to company protocol '{self.protocol_name}' (located in the \"Company Protocols\" knowledge base) re-distribute the problems with hashtag={self.problem_hashtag}." + + goal, info = super().setup_goal(page=page, config=config) + + return goal, info + + def _get_config(self) -> list[AbstractServiceNowTask]: + """ """ + navigate_to_protocol_subtask = [ + # Navigate to the KB + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Self-Service", + "module": "Knowledge", + "url": "/now/nav/ui/classic/params/target/%24knowledge.do", + }, + is_validated=False, + used_in_level_2=False, + ), + # Find the protocol for on-boarding a new user + KnowledgeBaseSearchTask( + instance=self.instance, + fixed_config={ + "alternative_answers": [], + "item": f"{self.protocol_name}", + "question": f"Can you find the '{self.protocol_name}' Protocol in the Knowledge Base?", + "value": "", + }, + is_validated=False, + used_in_level_2=False, + ), + ] + + find_most_and_least_busy_users_subtask = [ + # Navigate to the reports list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Reports", + "module": "Administration > All", + "url": "/now/nav/ui/classic/params/target/sys_report_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + WorkLoadBalancingMinMaxRetrievalTask( + instance=self.instance, + fixed_config={ + "url": "/now/nav/ui/classic/params/target/sys_report", + "chart_title": self.plot_title, + "chart_series": "", + "question": "min", + }, + is_validated=False, + used_in_level_2=True, + problem_hashtag=self.problem_hashtag, + ), + ] + + reassign_problem_subtask = [ + # Navigate to the hardware asset list + AllMenuTask( + instance=self.instance, + fixed_config={ + "application": "Problem", + "module": "All", + "url": "/now/nav/ui/classic/params/target/problem_list.do", + }, + is_validated=False, + used_in_level_2=True, + ), + # Filter the problems by assignee, and priority = lowest priority + # The existence of a lower priority problem is guaranteed + FilterProblemListForWorkLoadBalancingTask( + instance=self.instance, + fixed_config={ + "filter_columns": ["assigned_to", "priority"], + "filter_kind": "AND", + "filter_values": [ + f"{self.user_with_most_problems}", + f"{self.lowest_priority}", + ], + }, + is_validated=False, + used_in_level_2=True, + goal=f'Create a filter to find problems where \n - "Assigned to" is the user with the most problems assigned and \n - "Priority" is "{self.lowest_priority}".', + ), + # Assign a problem to the user with the least problems assigned to them + EditProblemTask( + instance=self.instance, + new_values={"assigned_to": f"{self.user_with_least_problems}"}, + record_sys_id=self.problem_to_edit_sys_id, + record_number=self.problem_to_edit_number, + is_validated=True, + used_in_level_2=True, + level=self.level, + ), + ] + + config = ( + navigate_to_protocol_subtask + + find_most_and_least_busy_users_subtask + + reassign_problem_subtask + ) + + return config + + def teardown(self) -> None: + # Delete the users + for user_sys_id in self.user_sys_ids: + record_exists = table_api_call( + instance=self.instance, + table="sys_user", + params={"sysparm_query": f"sys_id={user_sys_id}"}, + ) + if record_exists: + db_delete_from_table( + instance=self.instance, + table="sys_user", + sys_id=user_sys_id, + ) + # Delete the problems + for problem_sys_id in self.problem_sys_ids: + record_exists = table_api_call( + instance=self.instance, + table="problem", + params={"sysparm_query": f"sys_id={problem_sys_id}"}, + ) + if record_exists: + db_delete_from_table( + instance=self.instance, + table="problem", + sys_id=problem_sys_id, + ) + # Delete the report + db_delete_from_table( + instance=self.instance, + table="sys_report", + sys_id=self.report_sys_id, + ) + super().teardown() + + +class WorkloadBalancingSmallTask(WorkloadBalancingTask, HumanEvalTask): + def __init__(self, seed: int = None, instance: SNowInstance = None, level: int = 2) -> None: + super().__init__( + seed=seed, + instance=instance, + level=level, + min_users=2, + max_users=4, + max_problem_range=[4, 6], + mid_problem_range=[3, 4], + min_problem_range=[1, 3], + ) + + +class WorkloadBalancingMediumTask(WorkloadBalancingTask): + def __init__(self, seed: int = None, instance: SNowInstance = None, level: int = 2) -> None: + super().__init__( + seed=seed, + instance=instance, + level=level, + min_users=5, + max_users=7, + max_problem_range=[5, 6], + mid_problem_range=[3, 5], + min_problem_range=[1, 3], + ) + + +class WorkloadBalancingLargeTask(WorkloadBalancingTask): + def __init__(self, seed: int = None, instance: SNowInstance = None, level: int = 2) -> None: + super().__init__( + seed=seed, + instance=instance, + level=level, + min_users=8, + max_users=10, + max_problem_range=[5, 6], + mid_problem_range=[3, 5], + min_problem_range=[1, 3], + ) + + +local_vars = locals().copy() + +__TASKS__ = [ + var + for var in local_vars.values() + if isinstance(var, type) + and issubclass(var, WorkloadBalancingTask) + and var is not WorkloadBalancingTask +] diff --git a/src/browsergym/workarena/tasks/dashboard.py b/src/browsergym/workarena/tasks/dashboard.py index 84bb655..3c9fa2e 100644 --- a/src/browsergym/workarena/tasks/dashboard.py +++ b/src/browsergym/workarena/tasks/dashboard.py @@ -10,6 +10,9 @@ from urllib import parse from .base import AbstractServiceNowTask +from .comp_building_block import CompositionalBuildingBlockTask +from .utils.utils import check_url_suffix_match + from ..api.utils import table_api_call, table_column_info from ..config import ( DASHBOARD_RETRIEVAL_MINMAX_CONFIG_PATH, @@ -21,6 +24,7 @@ ) from ..instance import SNowInstance from .utils.string import share_tri_gram +from .utils.utils import check_url_suffix_match # XXX: Some notes on plot types # - We currently don't support maps because they are clickable and would require a more evolved cheat function @@ -33,10 +37,17 @@ class DashboardRetrievalTask(AbstractServiceNowTask, ABC): """ - def __init__(self, seed: int, instance: SNowInstance = None, fixed_config: dict = None) -> None: + def __init__( + self, seed: int = None, instance: SNowInstance = None, fixed_config: dict = None, **kwargs + ) -> None: super().__init__(seed=seed, instance=instance, start_rel_url="") self.iframe_id = "gsft_main" self.fixed_config = fixed_config + self.__dict__.update(kwargs) + + @abstractmethod + def all_configs(self) -> List[dict]: + pass @abstractmethod def all_configs(self) -> List[dict]: @@ -228,12 +239,6 @@ def _wait_for_ready(self, page: playwright.sync_api.Page) -> None: logging.debug("All plots loaded") def get_init_scripts(self) -> List[str]: - # Configure to page type - # ... extract URL suffix - url_suffix = parse.unquote( - parse.urlparse(self.config["url"].replace("%3F", "?")).path.split("/")[-1] - ) - return super().get_init_scripts() + [ "registerGsftMainLoaded();", f""" @@ -263,7 +268,28 @@ def get_init_scripts(self) -> List[str]: waLog('All charts loaded', 'loadAllCharts'); }}); }} - runInGsftMainOnlyAndProtectByURL(renderAllCharts, '{url_suffix}'); + // Run on both dashboard and reports pages + runInGsftMainOnlyAndProtectByURL(renderAllCharts, 'pa_dashboard.do'); + runInGsftMainOnlyAndProtectByURL(renderAllCharts, 'sys_report_template.do'); + """, + f""" + function purifyReportUIButtons() {{ + // Delete a lot of UI features that were causing issues due to the report refreshing without + // reloading the page. This makes the task easier, but it doesn't matter because we really + // want to evaluate retrieval and this doesn't prevent that. + document.querySelectorAll('[ng-click*="main.runReport"], #sidebar, #nlq-over-cb, #open-tree-navigation-button, .data-filtering-wrap').forEach(element => {{ + if (element && element.parentNode) {{ + element.parentNode.removeChild(element); + }} + }}); + document.addEventListener('click', function(event) {{ + event.stopPropagation(); + event.preventDefault(); + }}, true); + waLog('Purified report UI.', 'purifyReportUIButtons'); + }} + // Run it only on the reports page + runInGsftMainOnlyAndProtectByURL(purifyReportUIButtons, 'sys_report_template.do'); """, ] @@ -295,6 +321,12 @@ def setup_goal(self, page: playwright.sync_api.Page) -> Tuple[str | dict]: goal = f"What is the maximum value in {chart_locator}? Give me both the label and the count. If there are many, pick one." elif self.config["question"] == "min": goal = f"What is the minimum value in {chart_locator}? Give me both the label and the count. If there are many, pick one." + elif self.config["question"] == "mean": + goal = f"What is the average value in {chart_locator}? Round off to the next highest integer." + elif self.config["question"] == "median": + goal = f"What is the median value in {chart_locator}?" + elif self.config["question"] == "mode": + goal = f"What is the mode value in {chart_locator}?" else: raise NotImplementedError(f"Question type {self.config['question']} not supported") @@ -302,6 +334,34 @@ def setup_goal(self, page: playwright.sync_api.Page) -> Tuple[str | dict]: def cheat(self, page: playwright.sync_api.Page, chat_messages: list[str]) -> None: super().cheat(page, chat_messages) + # Check if the page is the report list view. If so, open the report + page_is_report_list_view = check_url_suffix_match( + page, "/now/nav/ui/classic/params/target/sys_report_list.do", self + ) + chart_title = self.config["chart_title"] + if page_is_report_list_view: + # Open the report + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + # Search for the report by title + frame.get_by_label("Search a specific field of the Reports list").select_option("Title") + search_input = frame.locator('input[aria-label="Search"]') + search_input.click() + search_input.fill(chart_title) + search_input.press("Enter") + page.wait_for_function( + "typeof window.gsft_main !== 'undefined' && window.gsft_main.WORKARENA_LOAD_COMPLETE" + ) + # Click on the chart preview to open it + frame.wait_for_selector(f'a[aria-label="Preview record: {chart_title}"]').click() + page.wait_for_timeout(1000) + page.keyboard.press("Enter") + # Now in the form view, wait for the page to load and click to view the report + page.wait_for_function( + "typeof window.gsft_main !== 'undefined' && window.gsft_main.WORKARENA_LOAD_COMPLETE" + ) + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + frame.get_by_text("View Report").first.click() + self._wait_for_ready(page) # Get the chart data @@ -345,6 +405,37 @@ def cheat(self, page: playwright.sync_api.Page, chat_messages: list[str]) -> Non chat_messages.append( {"message": f"{min_point['label']}, {min_point['count']}", "role": "assistant"} ) + elif self.config["question"] == "mean": + counts = [data["count"] for data in chart_data] + target_count = np.mean(counts) + chat_messages.append({"message": f"Mean / Average {target_count}", "role": "assistant"}) + elif self.config["question"] == "median": + counts = [data["count"] for data in chart_data] + target_count = np.median(counts) + chat_messages.append({"message": f"Median {target_count}", "role": "assistant"}) + elif self.config["question"] == "mode": + counts = [data["count"] for data in chart_data] + # We select the maximum value if there are two or more modes + frequencies = {} + for count in counts: + if count not in frequencies: + frequencies[count] = 1 + else: + frequencies[count] += 1 + sorted_frequencies = { + count: frequency + for count, frequency in sorted( + frequencies.items(), key=lambda item: item[1], reverse=True + ) + } + max_frequency = list(sorted_frequencies.values())[0] + max_frequencies = [ + count + for count, frequency in sorted_frequencies.items() + if frequency == max_frequency + ] + target_count = max(max_frequencies) + chat_messages.append({"message": f"Mode {target_count}", "role": "assistant"}) else: raise NotImplementedError(f"Question type \"{self.config['question']}\" not supported") @@ -352,12 +443,28 @@ def validate( self, page: playwright.sync_api.Page, chat_messages: list[str] ) -> Tuple[float, bool, str, dict]: super().validate(page, chat_messages) + + # Check if the page is in the right URL + logging.debug("Checking if the page is in the right URL to validate the task") + right_url = check_url_suffix_match(page, expected_url=self.start_url, task=self) + if not right_url: + return ( + 0, + False, + "", + { + "message": f"The page is not in the right URL to validate task {self.__class__.__name__}." + }, + ) + self._wait_for_ready(page) # Get the chart data + logging.debug("Extracting chart data") _, chart_data, _ = self._get_chart_by_title(page, self.config["chart_title"]) # Extract the series + logging.debug("Extracting the series") if len(chart_data) == 1: chart_data = chart_data[0]["data"] else: @@ -368,6 +475,7 @@ def validate( ][0] # Extract the agent's response + logging.debug("Extracting the agent's response") if chat_messages and chat_messages[-1]["role"] == "assistant": response = chat_messages[-1]["message"] else: @@ -379,6 +487,7 @@ def validate( ) # Extract all numbers mentioned by the agent + logging.debug("Extracting all numbers mentioned by the agent") # ... some value labels may contain numbers so we need to remove the labels from the response first labels = set([point["label"] for point in chart_data]) response_ = str(response) @@ -390,15 +499,29 @@ def validate( ) del response_ + if len(response_floats) == 0: + return ( + 0.0, + False, + "No number detected in the response.", + {"message": "No number detected in the response."}, + ) + # Validate the response + logging.debug("Validating the response based on the question type") if self.config["question"].startswith("value"): + logging.debug("The question is a value question") # if more than one number is in the prompt, there is necessarily a false positive if len(response_floats) > 1: error_msg = "Incorrect answer. More than one number detected in the response." return 0.0, True, error_msg, {"message": error_msg} + logging.debug( + f"Extracting expected format and label from question for validation: {self.config['question']}" + ) format = self.config["question"].split(";")[1].strip() label = self.config["question"].split(";")[2].strip() + logging.debug(f"Extracted format: {format}, label: {label}") expected_value = float( [ @@ -416,6 +539,7 @@ def validate( elif "max" in self.config["question"] or "min" in self.config["question"]: # Determine whether to find max or min based on configuration target_func = max if self.config["question"] == "max" else min + logging.debug(f"The question is a {str(target_func)} question") # Get the target count value (max or min) target_count = float(target_func(chart_data, key=lambda x: x["count"])["count"]) @@ -437,6 +561,33 @@ def validate( # If no correct point is mentioned in the response return 0.0, True, "Incorrect answer.", {"message": "Incorrect answer."} + # ... validate mean/median/mode responses + elif ( + "mean" in self.config["question"] + or "median" in self.config["question"] + or "mode" in self.config["question"] + ): + counts = [data["count"] for data in chart_data] + if self.config["question"] == "mean": + target_count = np.mean(counts) + elif self.config["question"] == "median": + target_count = np.median(counts) + elif self.config["question"] == "mode": + _vals, _counts = np.unique(counts, return_counts=True) + max_frequency_index = np.argmax(_counts) + target_count = -_vals[max_frequency_index] + + # if more than one number is in the prompt, there is necessarily a false positive + if len(response_floats) > 1: + error_msg = "Incorrect answer. More than one number detected in the response." + return 0.0, True, error_msg, {"message": error_msg} + + # Check if any of these points are mentioned in the response + if np.isclose(target_count, response_floats[0]): + return 1.0, True, "Nice work, thank you!", {"message": "Correct answer."} + + # If no correct point is mentioned in the response + return 0.0, True, "Incorrect answer.", {"message": "Incorrect answer."} else: raise NotImplementedError(f"Question type \"{self.config['question']}\" not supported") @@ -611,10 +762,39 @@ def all_configs(self): return json.load(open(REPORT_RETRIEVAL_MINMAX_CONFIG_PATH, "r")) +class ReportMeanMedianModeRetrievalTask(DashboardRetrievalTask, CompositionalBuildingBlockTask): + def all_configs(self): + return json.load(open(REPORT_RETRIEVAL_MINMAX_CONFIG_PATH, "r")) + + +class WorkLoadBalancingMinMaxRetrievalTask( + DashboardMinMaxRetrievalTask, CompositionalBuildingBlockTask +): + def all_configs(self): + return json.load(open(REPORT_RETRIEVAL_MINMAX_CONFIG_PATH, "r")) + + def setup_goal(self, page: playwright.sync_api.Page) -> Tuple[str | dict]: + super().setup_goal(page=page) + + # Configure task + # ... sample a configuration + self.config = ( + self.fixed_config if self.fixed_config else self.random.choice(self.all_configs()) + ) + # ... set start URL based on config + self.start_url = self.instance.snow_url + self.config["url"] + + goal = f"Create a filter to find reports whose title contains hashtag {self.problem_hashtag} and open the report." + goal += " From the report, identify the user with the most assigned problems and the user with the least assigned problems." + + return goal, {} + + __TASKS__ = [ var for var in locals().values() if isinstance(var, type) and issubclass(var, DashboardRetrievalTask) + and not issubclass(var, CompositionalBuildingBlockTask) and var is not DashboardRetrievalTask ] diff --git a/src/browsergym/workarena/tasks/form.py b/src/browsergym/workarena/tasks/form.py index b13a733..7ea3561 100644 --- a/src/browsergym/workarena/tasks/form.py +++ b/src/browsergym/workarena/tasks/form.py @@ -1,21 +1,29 @@ +import inspect import json import logging import playwright.sync_api import re +from collections import OrderedDict from english_words import get_english_words_set +from faker import Faker + +fake = Faker() from playwright.sync_api._generated import Page +from tenacity import retry, stop_after_delay, retry_if_exception_type from time import sleep from typing import List, Tuple from urllib import parse +from .base import AbstractServiceNowTask +from .comp_building_block import CompositionalBuildingBlockTask + from ..api.utils import ( db_delete_from_table, table_api_call, table_column_info, HTTPError, ) -from .base import AbstractServiceNowTask from ..config import ( SNOW_BROWSER_TIMEOUT, # Paths to the configuration files @@ -30,23 +38,60 @@ EXPECTED_INCIDENT_FORM_FIELDS_PATH, EXPECTED_PROBLEM_FORM_FIELDS_PATH, EXPECTED_USER_FORM_FIELDS_PATH, + EXPECTED_REQUEST_ITEM_FORM_FIELDS_PATH, ) from ..instance import SNowInstance from .utils.form import fill_text -from .utils.utils import check_url_suffix_match +from .utils.utils import check_url_suffix_match, prettyprint_enum ENGLISH_WORDS = list(get_english_words_set(["web2"])) class ServiceNowFormTask(AbstractServiceNowTask): + """ + Generic task for record manipulation (create/edit) in a table using a Glide form. + + Class attributes: + ----------------- + config_path: str + Path to the JSON file containing all possible configurations for the task. Defined in subclasses + expected_fields_path: str + Path to the JSON file containing all expected fields for the task. Defined in subclasses + + Parameters: + ----------------- + form_url: str + The URL of the form to use to create the record. + instance: SNowInstance + The instance on which to create the record. + extra_mandatory_fields: List + List of fields that should be marked as mandatory in the form (overrides the page specification). + unique_valued_fields: dict + Dictionary of fields that should have a unique value. Keys are the field names and values are functions + used to make the fields unique (e.g., appending self.unique). + fixed_config: dict + Configuration to use for the task. If provided, the task will use the provided configuration instead of + selecting a random one. See browsergym/workarena/data_files/task_configs/create_hardware_asset_task.json + for an example of a configuration file. + check_record_created: bool + Whether to check if the record is created in cheat. This step uses the localStorage to get the sys_id, which is None when creating multiple forms. Hence, we bypass this step in the cheat. + """ + + config_path = None + expected_fields_path = None + def __init__( self, - seed: int, - start_rel_url, + form_url: str, + table_label: str, instance: SNowInstance = None, extra_mandatory_fields: List = [], prohibited_fields: List = [], + unique_valued_fields: dict = {}, + fixed_config: dict = None, + check_record_created: bool = True, + seed: int = None, ) -> None: # The type of fields that we support interacting with self.supported_types = [ @@ -70,19 +115,55 @@ def __init__( # Prohibited fields: fields that we shouldn't interact with self.prohibited_fields = prohibited_fields + self.table_metadata = None + self.fields = None + self.mandatory_fields = None + self.optional_fields = None + + super().__init__(seed=seed, instance=instance, start_rel_url=form_url) + + self.form_url = form_url + + # Table pretty printed name + self.table_label = table_label + self.table_name = self.form_url.split("/")[-1].split(".do")[0] + + # Key in which the sys_id of the created record will be stored in the local storage + self.session_sys_id_field = f"{id(self)}.record_sys_id" + + # Fields that should have a unique value (will append them with a uuid) + self.unique_valued_fields = unique_valued_fields + + # Fixed configuration + # We set the task fields, template record and created sysids to allow for easy access in compositional task creation + self.fixed_config = fixed_config + self.template_record = None + self.task_fields = None + self.fields = None + self.protected_fields = None # Fields that should not be edited + if fixed_config is not None: + self._set_required_config_attributes(fixed_config) + + self.n_extra_fields = None + self.created_sysids = [] + if self.config_path: + self.all_configs = self.all_configs() + if self.expected_fields_path: + with open(self.expected_fields_path, "r") as f: + self.expected_fields = json.load(f) + self.check_record_created = check_record_created - super().__init__(seed=seed, instance=instance, start_rel_url=start_rel_url) + @classmethod + def all_configs(cls) -> List[dict]: + with open(cls.config_path, "r") as f: + return json.load(f) def _get_form(self, page): """ Loads a bunch of info about the form on a page into object variables - """ - # Extract Glide table information logging.debug("Extracting Glide table metadata") - # ... name of data table - self.table_name = page.evaluate(f"{self.form_js_selector}.getTableName()") # ... expand reference fields # XXX: We need to expand reference fields and the referenced field is missing from the # form's client-side info so we are going to use the meta API to get that info. @@ -107,6 +188,14 @@ def _get_form(self, page): }, )["result"][0]["label"].lower() + def _get_fields(self, page: Page) -> None: + """ + Get the form fields; split them into mandatory and optional + """ + page.wait_for_function( + f"typeof window.{self.js_prefix} !== 'undefined' && window.{self.js_prefix}.WORKARENA_LOAD_COMPLETE", + ) + # Get the form fields def is_field_visible(field): return page.evaluate( @@ -192,28 +281,43 @@ def _preprocess_fields(self, field, value): return value - def _wait_for_ready(self, page: Page) -> None: + def _wait_for_ready(self, page: Page, iframe_only=False) -> None: """ - Waits for the main iframe to be fully loaded. + Waits for the main iframe and APIs to be fully loaded + + Parameters: + ---------- + page: playwright.sync_api.Page + The page on which to wait for the iframe to be loaded + iframe_only: bool + If True, only wait for the iframe to be loaded. If False, also wait for the APIs to be available. """ logging.debug(f"Waiting for {self.js_prefix} to be fully loaded") - page.wait_for_function( - f"typeof window.{self.js_prefix} !== 'undefined' && window.{self.js_prefix}.WORKARENA_LOAD_COMPLETE", - ) + try: + page.wait_for_function( + f"typeof window.{self.js_prefix} !== 'undefined' && window.{self.js_prefix}.WORKARENA_LOAD_COMPLETE", + ) + except: + page.wait_for_load_state("networkidle") + return logging.debug(f"Detected {self.js_prefix} ready") - logging.debug("Waiting for Glide form API to be available") - page.wait_for_function(f"window.{self.form_js_selector}") - logging.debug("Detected Glide form API ready") + if not iframe_only: + logging.debug("Waiting for Glide form API to be available") + page.wait_for_function(f"window.{self.form_js_selector}") + logging.debug("Detected Glide form API ready") - logging.debug("Waiting for Glide tabs API to be available") - page.wait_for_function(f"typeof window.{self.js_prefix}.g_tabs2Sections !== 'undefined'") - logging.debug("Detected Glide tabs API ready") + logging.debug("Waiting for Glide tabs API to be available") + page.wait_for_function( + f"typeof window.{self.js_prefix}.g_tabs2Sections !== 'undefined'" + ) + logging.debug("Detected Glide tabs API ready") def get_init_scripts(self) -> List[str]: # Extract expected URL suffix url_suffix = parse.urlparse(self.start_url).path.split("/")[-1] + url_suffix = self.table_name # Add a few initialization scripts return super().get_init_scripts() + [ @@ -228,6 +332,45 @@ def get_init_scripts(self) -> List[str]: runInGsftMainOnlyAndProtectByURL(addFormMandatoryFields, '{url_suffix}'); """, + f""" + function patchSubmitButton() {{ + waLog('Attempting to override form submit function', 'patchSubmitButton'); + // Save the original function if it hasn't been saved yet + if(typeof old_gsftSubmit == 'undefined'){{ + old_gsftSubmit = new Function('return ' + gsftSubmit.toString())(); + waLog('Saved original submit function', 'patchSubmitButton'); + }} + + // Override the function to save the sys_id in the local storage + gsftSubmit = function(control, form, action_name) {{ + localStorage['{self.session_sys_id_field}'] = {self.js_api_forms}.getUniqueValue(); + old_gsftSubmit(control, form, action_name); + }}; + waLog('Patched submit function. All done.', 'patchSubmitButton'); + }} + + runInGsftMainOnlyAndProtectByURL(patchSubmitButton, '{url_suffix}'); + """, + # Ensure that only the expected fields are changed + f""" + function monitorChangeOnFields() {{ + let predefinedList = {json.dumps(self.protected_fields)}; + console.log('Predefined list: ' + predefinedList); + document.querySelectorAll("input, select, textarea").forEach((e) => {{ + // Get the field name - some fields are like incident.xyz.field_name + let fieldName = e.name.split('.').pop(); + if (!predefinedList.includes(fieldName)) {{ + e.addEventListener("change", () => {{ + window.WORKARENA_BAD_FIELD_CHANGED = true; + console.log("Field " + e.name + " changed and was not expected to."); + }}) + waLog('Added change listener to field ' + e.name, 'monitorChangeOnFields'); + }} + }}) + }} + + runInGsftMainOnlyAndProtectByURL(monitorChangeOnFields, '{url_suffix}'); + """, ] def start(self, page: Page) -> None: @@ -235,6 +378,177 @@ def start(self, page: Page) -> None: self._wait_for_ready(page) self._get_form(page) + def _fill_fields( + self, + page: Page, + iframe: playwright.sync_api.Frame, + task_fields: List[str], + update: bool = False, + ) -> None: + """ + Fill the fields in the form with the values from the template record. The fields to fill are specified in the + task_fields list. Update is a flag that indicates if the task is an update task. + """ + # XXX We need to ensure the table metadata as well as fields are set + # before we can proceed with the cheat function + if self.table_metadata is None: + self._get_form(page) + if self.fields is None: + self._get_fields(page) + + # From now on, we assume we are on the form page + self._wait_for_ready(page) + + # Retry on TypeError since in very rare occasions, element evaluates to null, which raises a TypeError + @retry( + stop=stop_after_delay(SNOW_BROWSER_TIMEOUT // 1000), + retry=retry_if_exception_type(TypeError), + ) + def show_field_tab(field): + """ + Finds the control that allows to show the section where a field is located + and clicks on it. + + """ + section = page.evaluate( + f"""() => {{ + const element = {self.form_js_selector}.getElement('{field}'); + const ancestors = element.ancestors(); + for (let ancestor of ancestors) {{ + // Ancestor IDs are of the form "section-
" + if (ancestor.id.startsWith('section-')) {{ + return ancestor.id; + }} + }} + return null; // Return null if no matching ancestor is found + }}""" + ) + section_id = section.split("-")[-1] + tab_sections = { + s.split(".")[-1]: i + for i, s in enumerate(page.evaluate(f"{self.js_prefix}.g_tabs2Sections.tabIDs")) + } + + # If the section is not in the tabs do nothing (it's probably the main section) + if section_id not in tab_sections: + return + + page.evaluate_handle( + f"""{self.js_prefix}.g_tabs2Sections.tabsTabs[ + {tab_sections[section_id]} + ].element""" + ).click(force=True) + + for field in task_fields: + # Get the field's input control + control = iframe.get_by_label( + page.evaluate(f"{self.form_js_selector}.getLabelOf('{field}')"), + exact=True, + ) + if control.count() > 1: + control = control.nth(0) + # If the field is in a section, click on its header to make it visible + show_field_tab(field) + + # Some fields are marked as string by the API but accept selection-based input + # We use the select tag condition to match these fields. Others are marked as integers. + if self.table_metadata[field]["type"] == "choice": + control.select_option(str(self.template_record[field])) + + # Checkboxes + elif self.table_metadata[field]["type"] == "boolean": + control.set_checked(1 if self.template_record[field] == "true" else 0) + + # Any text-based input + else: + fill_text( + page=page, + iframe=iframe, + input_field=control, + value=self.template_record[field], + ) + + # Click on the submit button + page.wait_for_timeout(1000) + if update: + iframe.locator("#sysverb_update").click() + else: + iframe.locator("#sysverb_insert").click() + + # Check if the record was created + if self.check_record_created: + # This does not work if multiple forms are created at once. The localStorage returns null after the first form + for attempt in range(5): + # in update tasks, the sys_id is already known as the asset is created from the start + if update: + sys_id = self.record_sys_id + else: + sys_id = page.evaluate("localStorage").get(self.session_sys_id_field, None) + + # Pull the record from the database + record = table_api_call( + instance=self.instance, + table=self.table_name, + params={ + "sysparm_query": f"sys_id={sys_id}", + "sysparm_display_value": True, + }, + )["result"] + if len(record) > 0: + break + page.wait_for_timeout(1500) + if attempt == 4: + raise ValueError("The record was not created.") + + def _set_required_config_attributes(self, config: dict) -> None: + """ + Set the required attributes for the task configuration. + """ + # XXX Warning: Some subclasses may expect a specific order of elements + self.template_record = config["template_record"] + for f, func in self.unique_valued_fields.items(): + self.template_record[f] = func(self.template_record[f]) + self.task_fields = config["task_fields"] + + def get_new_field_value(self, field: str, template_record: dict, table_metadata: dict) -> str: + """ + Generate a new value for a field based on the field type. + """ + new_value = template_record[ + field + ] # Default to the template value in case the task field is not of the supported types + if field in self.unique_valued_fields: + return new_value + if "choices" in table_metadata[field]: + if ( + # ... if the field has choices that are not available in the UI + template_record[field] not in table_metadata[field]["choices"].values() + or + # ... avoid empty values if there are other choices + ( + (template_record[field] is None or template_record[field] == "") + and len(table_metadata[field]["choices"]) > 1 + ) + ): + # XXX: We skip empty-string values because 1) they are not really interesting to + # ask for since the agent doesn't have to do anything. They also cause issues + # in the validation since they don't get saved properly to the database. + choices = [v for k, v in table_metadata[field]["choices"].items() if k != ""] + new_value = self.random.choice(choices) + elif table_metadata[field]["type"] in self.string_types: + # ... if the field is a string, we want to make sure that it's not empty + + if table_metadata[field]["type"] == "string": + new_value = " ".join(self.random.choice(ENGLISH_WORDS, size=5)) + elif table_metadata[field]["type"] == "email": + new_value = f"{'.'.join(self.random.choice(ENGLISH_WORDS, size=2))}@workarena.com" + elif table_metadata[field]["type"] == "ph_number": + new_value = ( + f"(514) {self.random.randint(100, 999)}-{self.random.randint(1000, 9999)}" + ) + + return new_value + class GenericNewRecordTask(ServiceNowFormTask): """ @@ -242,32 +556,17 @@ class GenericNewRecordTask(ServiceNowFormTask): Parameters: ----------- - form_url: str - The URL of the form to use to create the record. - instance: SNowInstance - The instance on which to create the record. - extra_mandatory_fields: List - List of fields that should be marked as mandatory in the form (overrides the page specification). - unique_valued_fields: dict - Dictionary of fields that should have a unique value. Keys are the field names and values are functions - used to make the fields unique (e.g., appending self.unique). min_fields: int Minimum number of fields to fill (except if mandatory is more). max_fields: int Maximum number of fields to fill (except if mandatory is more). - fixed_config: dict - Configuration to use for the task. If provided, the task will use the provided configuration instead of - selecting a random one. See browsergym/workarena/data_files/task_configs/create_hardware_asset_task.json - for an example of a configuration file. - config_path: - The path to the JSON file containing all configurations for the task. Provided by subclasses - expected_fields_path: - The path to the JSON file containing all expected fields for the task. Provided by subclasses """ + config_path = None + expected_fields_path = None + def __init__( self, - seed: int, form_url: str, table_label: str, instance: SNowInstance = None, @@ -277,70 +576,26 @@ def __init__( min_fields: int = 5, max_fields: int = None, fixed_config: dict = None, - config_path: str = None, - expected_fields_path: str = None, + seed: int = None, + check_record_created: bool = True, ) -> None: super().__init__( seed=seed, + form_url=form_url, + table_label=table_label, instance=instance, - start_rel_url=form_url, extra_mandatory_fields=extra_mandatory_fields, prohibited_fields=prohibited_fields, + unique_valued_fields=unique_valued_fields, + fixed_config=fixed_config, + check_record_created=check_record_created, ) - self.form_url = form_url - - # Table pretty printed name - self.table_label = table_label - - # Key in which the sys_id of the created record will be stored in the local storage - self.session_sys_id_field = f"{id(self)}.record_sys_id" - - # Fields that should have a unique value (will append them with a uuid) - self.unique_valued_fields = unique_valued_fields - # Maximum number of fields to fill (except if mandatory is more) self.min_fields = min_fields self.max_fields = 999999999 if max_fields is None else max_fields - - # Fixed configuration - self.fixed_config = fixed_config - - self.n_extra_fields = None - self.template_record = None - self.created_sysids = None - if config_path: - with open(config_path, "r") as f: - self.all_configs = json.load(f) - if expected_fields_path: - with open(expected_fields_path, "r") as f: - self.expected_fields = json.load(f) - - def get_init_scripts(self) -> List[str]: - # Extract expected URL suffix - url_suffix = parse.urlparse(self.start_url).path.split("/")[-1] - - # Add a few initialization scripts - return super().get_init_scripts() + [ - f""" - function patchSubmitButton() {{ - waLog('Attempting to override form submit function', 'patchSubmitButton'); - // Save the original function if it hasn't been saved yet - if(typeof old_gsftSubmit == 'undefined'){{ - old_gsftSubmit = new Function('return ' + gsftSubmit.toString())(); - waLog('Saved original submit function', 'patchSubmitButton'); - }} - - // Override the function to save the sys_id in the local storage - gsftSubmit = function(control, form, action_name) {{ - localStorage['{self.session_sys_id_field}'] = {self.js_api_forms}.getUniqueValue(); - old_gsftSubmit(control, form, action_name); - }}; - waLog('Patched submit function. All done.', 'patchSubmitButton'); - }} - - runInGsftMainOnlyAndProtectByURL(patchSubmitButton, '{url_suffix}'); - """ - ] + self.page_on_form_view = ( + False # Indicates if the page is on the form view; used in validation + ) def setup_goal(self, page: Page) -> tuple[str, dict]: super().setup_goal(page=page) @@ -348,16 +603,14 @@ def setup_goal(self, page: Page) -> tuple[str, dict]: # Get the task configuration assert self.all_configs is not None, "No configuration available for the task." config = self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) - self.template_record = config["template_record"] - for f, func in self.unique_valued_fields.items(): - self.template_record[f] = func(self.template_record[f]) - self.task_fields = config["task_fields"] - self.created_sysids = [] - + # If fixed_config is not None we already set the required attributes in the constructor + if self.fixed_config is None: + self._set_required_config_attributes(config) + self.protected_fields = self.task_fields # Generate the goal goal = ( f"Create a new {self.table_label} with " - + " and ".join( + + prettyprint_enum( [ f'a value of "{self.template_record[f]}"' + f' for field "{config["fields"][f]}"' @@ -439,37 +692,8 @@ def _generate_random_config(self, page: Page) -> None: # Replace some field values for f in self.fields: - if "choices" in self.table_metadata[f]: - if ( - # ... if the field has choices that are not available in the UI - self.template_record[f] not in self.table_metadata[f]["choices"].values() - or - # ... avoid empty values if there are other choices - ( - (self.template_record[f] is None or self.template_record[f] == "") - and len(self.table_metadata[f]["choices"]) > 1 - ) - ): - # XXX: We skip empty-string values because 1) they are not really interesting to - # ask for since the agent doesn't have to do anything. They also cause issues - # in the validation since they don't get saved properly to the database. - choices = [v for k, v in self.table_metadata[f]["choices"].items() if k != ""] - self.template_record[f] = self.random.choice(choices) - elif self.table_metadata[f]["type"] in self.string_types: - # ... if the field is a string, we want to make sure that it's not empty - if self.template_record[f] == "": - if self.table_metadata[f]["type"] == "string": - self.template_record[f] = " ".join( - self.random.choice(ENGLISH_WORDS, size=5) - ) - elif self.table_metadata[f]["type"] == "email": - self.template_record[f] = ( - f"{'.'.join(self.random.choice(ENGLISH_WORDS, size=2))}@workarena.com" - ) - elif self.table_metadata[f]["type"] == "ph_number": - self.template_record[f] = ( - f"(514) {self.random.randint(100, 999)}-{self.random.randint(1000, 9999)}" - ) + new_value = self.get_new_field_value(f, self.template_record, self.table_metadata) + self.template_record[f] = new_value # Make sure the value satisfies the max length for the field self.template_record = { @@ -497,84 +721,55 @@ def _generate_random_config(self, page: Page) -> None: info = {} return goal, info + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + class_name = self.__class__.__name__ + class_name = class_name.replace("Create", "").replace("Task", "") + + # Split the words + words = re.findall(r"[A-Z][^A-Z]*", class_name) + class_name_formatted = " ".join(words) + table_metadata = table_column_info(instance=self.instance, table=self.table_name) + # pretty field names that are displayed to the user + task_fields = [] + for field in self.task_fields: + # In feasible tasks, the fields are always present + if field in table_metadata: + field_name = table_metadata[field]["label"] + # In infeasible tasks, the fields are absent from table_metadata + else: + field_name = " ".join(field.split("_")).capitalize() + + task_fields.append(field_name) + + field_values = [self.template_record[field] for field in self.task_fields] + current_task_info = dict(zip(task_fields, field_values)) + task_info = f"- Create a {class_name_formatted} with the following information: \n" + for field, value in current_task_info.items(): + task_info += f" - {field}: {value} \n" + + return task_info + def cheat(self, page: Page, chat_messages: list[str]) -> None: super().cheat(page=page, chat_messages=chat_messages) - self._wait_for_ready(page) + # If we are on the list view of the table, click on the "New" button + self._wait_for_ready(page, iframe_only=True) iframe = page.frame_locator(f'iframe[name="{self.js_prefix}"]') - - from tenacity import retry, stop_after_delay, retry_if_exception_type - - # Retry on TypeError since in very rare occasions, element evaluates to null, which raises a TypeError - @retry( - stop=stop_after_delay(SNOW_BROWSER_TIMEOUT // 1000), - retry=retry_if_exception_type(TypeError), - ) - def show_field_tab(field): - """ - Finds the control that allows to show the section where a field is located - and clicks on it. - - """ - section = page.evaluate( - f"""() => {{ - const element = {self.form_js_selector}.getElement('{field}'); - const ancestors = element.ancestors(); - for (let ancestor of ancestors) {{ - // Ancestor IDs are of the form "section-
" - if (ancestor.id.startsWith('section-')) {{ - return ancestor.id; - }} - }} - return null; // Return null if no matching ancestor is found - }}""" - ) - section_id = section.split("-")[-1] - tab_sections = { - s.split(".")[-1]: i - for i, s in enumerate(page.evaluate(f"{self.js_prefix}.g_tabs2Sections.tabIDs")) - } - - # If the section is not in the tabs do nothing (it's probably the main section) - if section_id not in tab_sections: - return - - page.evaluate_handle( - f"""{self.js_prefix}.g_tabs2Sections.tabsTabs[ - {tab_sections[section_id]} - ].element""" - ).click(force=True) - - for field in self.task_fields: - # Get the field's input control - control = iframe.get_by_label( - page.evaluate(f"{self.form_js_selector}.getLabelOf('{field}')"), - exact=True, - ) - - # If the field is in a section, click on its header to make it visible - show_field_tab(field) - - # Some fields are marked as string by the API but accept selection-based input - # We use the select tag condition to match these fields. Others are marked as integers. - if self.table_metadata[field]["type"] == "choice": - control.select_option(str(self.template_record[field])) - - # Checkboxes - elif self.table_metadata[field]["type"] == "boolean": - control.set_checked(1 if self.template_record[field] == "true" else 0) - - # Any text-based input - else: - fill_text( - page=page, - iframe=iframe, - input_field=control, - value=self.template_record[field], - ) - - # Click on the submit button - page.wait_for_timeout(1000) - iframe.locator("#sysverb_insert").click() + url = parse.urlparse(parse.unquote(self.page.evaluate("() => window.location.href"))) + if url.path.endswith("_list.do"): + # click on the sysverb_new button + with page.expect_navigation(): + iframe.locator("#sysverb_new").click() + iframe = page.frame_locator(f'iframe[name="{self.js_prefix}"]') + # On the change request page, additional steps need to be taken to open the form + if self.table_label == "change request": + self._wait_for_ready(page, iframe_only=True) + iframe.get_by_label("All").click() + iframe.get_by_text("Normal").first.click() + self._fill_fields(page, iframe, self.task_fields) def validate( self, page: playwright.sync_api.Page, chat_messages: list[str] @@ -587,8 +782,8 @@ def validate( that are not part of the task. """ - # check that the page is at the right url - right_url = check_url_suffix_match(page, expected_url=self.start_url, task=self) + + right_url = self._page_on_right_url(page) if not right_url: return ( 0, @@ -598,6 +793,24 @@ def validate( "message": f"The page is not in the right URL to validate task {self.__class__.__name__}." }, ) + protected_field_changed = page.evaluate( + "() => window.gsft_main.WORKARENA_BAD_FIELD_CHANGED" + ) + if protected_field_changed: + return ( + 0, + True, + "", + {"message": "Some fields outside of the task scope have been changed."}, + ) + if self.table_metadata is None and self.page_is_form_view: + # XXX We need to ensure the table metadata as well as fields are set + # before we can proceed with the cheat function + self._wait_for_ready(page, iframe_only=True) + self._get_form(page) + if self.fields is None and self.page_is_form_view: + self._get_fields(page) + # Retrieve the created record's sys_id from the session storage sys_id = page.evaluate("localStorage").get(self.session_sys_id_field, None) @@ -615,11 +828,10 @@ def validate( # This is used to clean up the database after the task is completed. self.created_sysids.append(sys_id) - # Short sleep to make sure the data is saved in the DB - # TODO: improve this (noted in issue 291) - sleep(3) - # Pull the record from the database + # XXX: It's possible that the record is not found, e.g., if form submission was rejected due to client-side + # validation errors. In this case, we should not raise an error and simply consider that no record was + # created. This is non-terminal for the task. record = table_api_call( instance=self.instance, table=self.table_name, @@ -627,6 +839,9 @@ def validate( "sysparm_query": f"sys_id={sys_id}", "sysparm_display_value": True, }, + wait_for_record=True, + max_retries=20, # Wait up to 10 seconds + raise_on_wait_expired=False, )["result"] # This can happen if the form was submitted but was rejected due to invalid inputs (e.g., missing mandatory fields) @@ -663,9 +878,31 @@ def validate( {"message": error_msg}, ) - return 1, True, "Nice work, thank you!", {"message": "The record was successfully created."} + return ( + 1, + True, + "Nice work, thank you!", + {"message": "The record was successfully created."}, + ) + + def _page_on_right_url(self, page: Page) -> bool: + """Checks if the page is on the right URL for validation + sets the page_on_form_view attribute""" + page.wait_for_load_state("domcontentloaded") + self._wait_for_ready(page, iframe_only=True) + # check that the page is at the right url + list_url = self.start_url.replace(".do", "_list.do") # list view of records + # Check whether we are in the form or list view + self.page_is_form_view = check_url_suffix_match( + page, expected_url=self.start_url, task=self + ) + page_is_list_view = check_url_suffix_match(page, expected_url=list_url, task=self) + + right_url = self.page_is_form_view or page_is_list_view + + return right_url def teardown(self) -> None: + self._wait_for_ready(self.page, iframe_only=True) # Retrieve the current record's sys_id from the session storage sys_id = self.page.evaluate("localStorage").get(self.session_sys_id_field, None) @@ -685,13 +922,313 @@ def teardown(self) -> None: pass +class EditRecordTask(ServiceNowFormTask, CompositionalBuildingBlockTask): + """ + Generic task to edit an existing record in a table using a Glide form. + Class Attributes + ---------------- + config_path: str + The path to the JSON file containing all configurations for the task. Defined by subclasses + expected_fields_path: str + The path to the JSON file containing all expected fields for the task. Defined by subclasses + Args + ---- + form_url: str + The URL of the form to use to edit the record. + table_label: str + The pretty-printed name of the table. + instance: SNowInstance + The instance on which to edit the record. + extra_mandatory_fields: List + List of fields that should be marked as mandatory in the form (overrides the page specification). + prohibited_fields: List + List of fields that should not be edited. + unique_valued_fields: dict + Dictionary of fields that should have a unique value. Keys are the field names and values are functions + used to make the fields unique (e.g., appending self.unique). + fixed_config: dict + Configuration to use for the task. If provided, the task will use the provided configuration instead of + a randomly selected one + record_sys_id: str + The sys_id of the record to edit. If provided, the task will edit this record instead of creating a new one. + record_number: str + The number of the record to edit. If provided, the task's cheat will select records based on it rather than picking the first element of the list. + new_values: dict + Dictionary mapping fields to their new values. These are values that will be used to either replace the current + values in the record or add them to the record if they are not already present. + """ + + def __init__( + self, + form_url: str, + table_label: str, + instance: SNowInstance = None, + extra_mandatory_fields: List = [], + prohibited_fields: List = [], + unique_valued_fields: dict = {}, + fixed_config: dict = None, + record_sys_id: str = None, + record_number: str = None, + new_values: dict = None, + seed: int = None, + ) -> None: + super().__init__( + seed=seed, + form_url=form_url, + table_label=table_label, + instance=instance, + extra_mandatory_fields=extra_mandatory_fields, + prohibited_fields=prohibited_fields, + unique_valued_fields=unique_valued_fields, + fixed_config=fixed_config, + ) + # sys_id of the record that will be edited + self.record_sys_id = record_sys_id + self.record_number = record_number + self.delete_record_on_teardown = False + self.new_values = new_values # dict mapping fields to their new values + # If the record sys_id is provided, the task will fetch its template record and task fields + if self.record_sys_id is not None: + fixed_config = {} + template_record = table_api_call( + instance=self.instance, + table=self.table_name, + params={ + "sysparm_query": f"sys_id={self.record_sys_id}", + }, + )["result"][0] + fixed_config["template_record"] = template_record + fixed_config["task_fields"] = list(self.new_values.keys()) + table_info = table_column_info(instance=self.instance, table=self.table_name) + fixed_config["fields"] = {f: table_info[f]["label"] for f in self.new_values.keys()} + + self.fixed_config = fixed_config + + def setup_goal(self, page: Page) -> tuple[str, dict]: + super().setup_goal(page=page) + + # Get the task configuration + config = self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) + + # If fixed_config is not None we already set the required attributes in the constructor + # If record_sys_id is not None, the required attributes are not set in the constructor either + if self.fixed_config is None or self.record_sys_id is not None: + self._set_required_config_attributes(config) + + # Make the new values unique if needed + for f, func in self.unique_valued_fields.items(): + if f in self.new_values: + self.new_values[f] = func(self.new_values[f]) + + self.protected_fields = list(self.new_values.keys()) + if self.record_sys_id is None: + self._create_record() + self.delete_record_on_teardown = True + # Replace the values in the template record + for f, v in self.new_values.items(): + self.template_record[f] = v + self.start_url = f"{self.start_url}%3Fsys_id%3D{self.record_sys_id}" + + # Generate the goal + goal = self.get_pretty_printed_description() + + info = {} + + return goal, info + + def _create_record(self) -> None: + """Create a record to edit.""" + # Data to create the record + data = {} + for field in self.template_record: + value = self.template_record[field] + if type(value) == dict: + value = value["display_value"] + # Skip sys fields as they are not editable + if not value or "sys" in field: + continue + data[field] = value + + result = table_api_call( + instance=self.instance, + table=self.table_name, + data=json.dumps(data), + method="POST", + ) + self.record_sys_id = result["result"]["sys_id"] + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + class_name = self.__class__.__name__ + class_name = class_name.replace("Edit", "").replace("Task", "") + # Split the words + words = re.findall(r"[A-Z][^A-Z]*", class_name) + table_metadata = table_column_info(instance=self.instance, table=self.table_name) + task_fields = [ + table_metadata[field]["label"] for field in self.new_values + ] # pretty field names that are displayed to the user + field_values = [self.template_record[field] for field in self.new_values] + current_task_info = dict(zip(task_fields, field_values)) + # In L3, this is part of an enumeration + task_info = "- " if self.level == 3 else "" + + task_info += ( + f"Edit the {self.table_label} record by replacing the value of " + + prettyprint_enum( + [ + f' field "{field}"' + f' with value "{value}"' + for field, value in current_task_info.items() + ] + ) + + "." + ) + + return task_info + + def cheat(self, page: Page, chat_messages: list[str]) -> None: + super().cheat(page=page, chat_messages=chat_messages) + self._wait_for_ready(page, iframe_only=True) + iframe = page.frame_locator(f'iframe[name="{self.js_prefix}"]') + url = parse.urlparse(parse.unquote(self.page.evaluate("() => window.location.href"))) + + # Open the record preview, then the record + if url.path.endswith("_list.do"): + # If the record number is provided, click on the record with that number + if self.record_number: + iframe.locator(f"[aria-label='Preview record: {self.record_number}']").click() + # ....otherwise, click on the first record + else: + iframe.locator("td").get_by_role("button").first.click() + page.wait_for_timeout(500) + + iframe.get_by_text("Open Record").click() + page.wait_for_function( + "typeof window.gsft_main !== 'undefined' && window.gsft_main.WORKARENA_LOAD_COMPLETE" + ) + page.wait_for_timeout(1000) + self._fill_fields(page, iframe, self.new_values.keys(), update=True) + + def validate( + self, page: playwright.sync_api.Page, chat_messages: list[str] + ) -> Tuple[float, bool, str, dict]: + """ + Caveat: we check only if the expected fields have the right value. We don't Check + if there are extra fields that shouldn't be there. We could have issues + matching other fields since calculation rules may have changed through time. + Maybe we should assign a random value from our list of choices to the fields + that are not part of the task. + + """ + page.wait_for_load_state("domcontentloaded") + # check that the page is at the right url + list_url = self.start_url.replace(".do", "_list.do") # list view of records + # Check whether we are in the form or list view + page_is_form_view = check_url_suffix_match(page, expected_url=self.start_url, task=self) + page_is_list_view = check_url_suffix_match(page, expected_url=list_url, task=self) + right_url = page_is_form_view or page_is_list_view + if not right_url: + return ( + 0, + False, + "", + { + "message": f"The page is not in the right URL to validate task {self.__class__.__name__}." + }, + ) + self._wait_for_ready(page, iframe_only=True) + protected_field_changed = page.evaluate( + "() => window.gsft_main.WORKARENA_BAD_FIELD_CHANGED" + ) + if protected_field_changed: + return ( + 0, + True, + "", + {"message": "Some fields outside of the task scope have been changed."}, + ) + if self.table_metadata is None: + # XXX We need to ensure the table metadata as well as fields are set + # before we can proceed with the cheat function + self._wait_for_ready(page, iframe_only=True) + self._get_form(page) + if self.fields is None and page_is_form_view: + self._get_fields(page) + + # Pull the record from the database + record = table_api_call( + instance=self.instance, + table=self.table_name, + params={ + "sysparm_query": f"sys_id={self.record_sys_id}", + "sysparm_display_value": True, + }, + wait_for_record=True, + )["result"] + + # This can happen if the form was submitted but was rejected due to invalid inputs (e.g., missing mandatory fields) + if len(record) == 0: + logging.info( + "The record was not found in the database. Perhaps it was deleted." + + self.record_sys_id, + ) + return ( + 0, + True, + "", + {"message": "The record was not found in the database. Perhaps it was deleted."}, + ) + + # Extract display values for reference fields + record = { + f: v if not isinstance(v, dict) else v["display_value"] for f, v in record[0].items() + } + + # Check that the record matches the expected values + for f in self.new_values.keys(): + if "sys_" in f: + continue + if record[f] != self.template_record[f]: + logging.info( + f'The field "{self.table_metadata[f]["label"]}" has the wrong value. Expected: "{self.template_record[f]}", got: "{record[f]}".' + ) + error_msg = f'The field "{self.table_metadata[f]["label"]}" has the wrong value.' + return ( + 0, + False, + error_msg, + {"message": error_msg}, + ) + + return ( + 1, + True, + "Nice work, thank you!", + {"message": "The record was successfully edited."}, + ) + + def teardown(self) -> None: + # Delete the record created for the task + if self.delete_record_on_teardown: + db_delete_from_table( + instance=self.instance, sys_id=self.record_sys_id, table=self.table_name + ) + + class CreateChangeRequestTask(GenericNewRecordTask): """ Task to create a new change request in the system. """ - def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: + config_path = CREATE_CHANGE_REQUEST_CONFIG_PATH + expected_fields_path = EXPECTED_CHANGE_REQUEST_FORM_FIELDS_PATH + + def __init__( + self, seed: int = None, instance=None, fixed_config: dict = None, **kwargs + ) -> None: super().__init__( seed=seed, instance=instance, @@ -699,18 +1236,43 @@ def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: table_label="change request", prohibited_fields=["chg_model", "state"], fixed_config=fixed_config, - config_path=CREATE_CHANGE_REQUEST_CONFIG_PATH, - expected_fields_path=EXPECTED_CHANGE_REQUEST_FORM_FIELDS_PATH, + ) + self.__dict__.update(kwargs) + + def _page_on_right_url(self, page: playwright.sync_api.Page) -> bool: + """ + The change request form lands in a view different from the list view. We need to check for this as well. + """ + right_url = super()._page_on_right_url(page) + # Change request creation leads to a different page when in comp task; we need to check this case as well + change_request_landing_page = "/now/nav/ui/classic/params/target/sn_chg_model_ui_landing.do" + page_is_change_landing = ( + check_url_suffix_match(page, expected_url=change_request_landing_page, task=self) + if self.table_label == "change request" + else False ) + right_url = right_url or page_is_change_landing + + return right_url + class CreateIncidentTask(GenericNewRecordTask): """ Task to create a new incident in the system. - """ - def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: + config_path = CREATE_INCIDENT_CONFIG_PATH + expected_fields_path = EXPECTED_INCIDENT_FORM_FIELDS_PATH + + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + check_record_created=True, + **kwargs, + ) -> None: super().__init__( seed=seed, instance=instance, @@ -718,9 +1280,9 @@ def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: table_label="incident", prohibited_fields=["state"], fixed_config=fixed_config, - config_path=CREATE_INCIDENT_CONFIG_PATH, - expected_fields_path=EXPECTED_INCIDENT_FORM_FIELDS_PATH, + check_record_created=check_record_created, ) + self.__dict__.update(kwargs) class CreateHardwareAssetTask(GenericNewRecordTask): @@ -729,7 +1291,12 @@ class CreateHardwareAssetTask(GenericNewRecordTask): """ - def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: + config_path = CREATE_HARDWARE_CONFIG_PATH + expected_fields_path = EXPECTED_HARDWARE_FORM_FIELDS_PATH + + def __init__( + self, seed: int = None, instance=None, fixed_config: dict = None, **kwargs + ) -> None: super().__init__( seed=seed, instance=instance, @@ -744,9 +1311,8 @@ def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: ], unique_valued_fields={"serial_number": lambda x: f"SN-{self.unique_id}"}, fixed_config=fixed_config, - config_path=CREATE_HARDWARE_CONFIG_PATH, - expected_fields_path=EXPECTED_HARDWARE_FORM_FIELDS_PATH, ) + self.__dict__.update(kwargs) class CreateProblemTask(GenericNewRecordTask): @@ -755,7 +1321,17 @@ class CreateProblemTask(GenericNewRecordTask): """ - def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: + config_path = CREATE_PROBLEM_CONFIG_PATH + expected_fields_path = EXPECTED_PROBLEM_FORM_FIELDS_PATH + + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + check_record_created=True, + **kwargs, + ) -> None: super().__init__( seed=seed, instance=instance, @@ -763,13 +1339,13 @@ def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: table_label="problem", prohibited_fields=["state", "first_reported_by_task"], fixed_config=fixed_config, - config_path=CREATE_PROBLEM_CONFIG_PATH, - expected_fields_path=EXPECTED_PROBLEM_FORM_FIELDS_PATH, + check_record_created=check_record_created, # TODO: The last field is disabled because somehow the value is not in the autocomplete # list even though it's in the database. I'm not sure why. It doesn't matter much # since in the future we'll pre-generate tasks and keep only the ones where the # cheat function works. ) + self.__dict__.update(kwargs) class CreateUserTask(GenericNewRecordTask): @@ -778,24 +1354,240 @@ class CreateUserTask(GenericNewRecordTask): """ - def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: + config_path = CREATE_USER_CONFIG_PATH + expected_fields_path = EXPECTED_USER_FORM_FIELDS_PATH + + def __init__( + self, seed: int = None, instance=None, fixed_config: dict = None, **kwargs + ) -> None: super().__init__( seed=seed, instance=instance, form_url="/now/nav/ui/classic/params/target/sys_user.do", table_label="user", extra_mandatory_fields=["user_name", "first_name", "last_name", "email"], - unique_valued_fields={"user_name": lambda x: str(hash(x + self.unique_id))}, + # XXX We use an OrderedDict to ensure that the fields are filled in the right order as the email requires the first and last name + unique_valued_fields=OrderedDict( + [ + ("first_name", lambda x: fake.first_name() + "-" + fake.first_name()), + ("last_name", lambda x: fake.last_name() + "-" + fake.last_name()), + ("user_name", lambda x: str(abs(hash(x + self.unique_id)))), + ( + "email", + lambda x: self.template_record["first_name"].lower() + + "." + + self.template_record["last_name"].lower() + + "@workarena.com", + ), + ] + ), fixed_config=fixed_config, - config_path=CREATE_USER_CONFIG_PATH, - expected_fields_path=EXPECTED_USER_FORM_FIELDS_PATH, ) + self.__dict__.update(kwargs) + + +class EditHardwareAssetTask(EditRecordTask): + """ + Task to create a new user in the system. + + """ + + config_path = CREATE_HARDWARE_CONFIG_PATH + expected_fields_path = EXPECTED_HARDWARE_FORM_FIELDS_PATH + + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + record_sys_id: str = None, + new_values: dict = None, + **kwargs, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + form_url="/now/nav/ui/classic/params/target/alm_hardware.do", + table_label="hardware asset", + prohibited_fields=["install_status"], + unique_valued_fields={"serial_number": lambda x: f"SN-{self.unique_id}"}, + new_values=new_values, + fixed_config=fixed_config, + record_sys_id=record_sys_id, + ) + if self.new_values is None: + self.new_values = {"department": "Finance"} + self.__dict__.update(kwargs) + + +class EditProblemTask(EditRecordTask): + """ + Task to edit a problem in the system. + + """ + + expected_fields_path = EXPECTED_PROBLEM_FORM_FIELDS_PATH + + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + new_values: dict = None, + record_sys_id: str = None, + record_number: str = None, + **kwargs, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + form_url="/now/nav/ui/classic/params/target/problem.do", + table_label="problem", + prohibited_fields=["state", "first_reported_by_task"], + new_values=new_values, + fixed_config=fixed_config, + record_sys_id=record_sys_id, + record_number=record_number, + ) + if self.new_values is None: + self.new_values = {"assigned_to": ""} + self.__dict__.update(kwargs) + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + if self.level == 2: + description = "Re-assign a lowest priority problem from the user with the most assigned problems to the user with the least assigned problems." + return description + else: + return "" + + +class EditChangeRequestScheduleTask(EditRecordTask): + """Task to edit an existing change request's empty schedule (start and end dates).""" + + expected_fields_path = EXPECTED_CHANGE_REQUEST_FORM_FIELDS_PATH + + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + new_values: dict = None, + record_sys_id: str = None, + skip_description: bool = False, + goal_type: str = "base", + level: int = 2, + **kwargs, + ) -> None: + """ + args: + ----- + skip_description: bool + Whether to skip the description field in the change request. Used in comp tasks when this class is used multiple times. + goal_type: str + Choice of "base", "priority", "tight", "tight priority". The type of goal to generate. Used in compositional tasks. + level: int + The level of the compositional task. Used in compositional tasks. + """ + super().__init__( + seed=seed, + instance=instance, + form_url="/now/nav/ui/classic/params/target/change_request.do", + table_label="change request", + prohibited_fields=["chg_model", "state"], + new_values=new_values, + fixed_config=fixed_config, + record_sys_id=record_sys_id, + ) + self.skip_description = skip_description + self.goal_type = goal_type + self.level = level + self.__dict__.update(kwargs) + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in compositional tasks. + """ + if self.skip_description or self.level == 3: + return "" + elif self.goal_type == "base": + task_info = "Edit the schedule of the change requests by setting the start and end dates so that the change requests do not overlap. There should not be more than one day between conescutive change requests in the schedule." + elif self.goal_type == "priority": + task_info = "Edit the schedule of the change requests by setting the start and end dates so that the change requests do not overlap. There should not be more than one day between conescutive change requests in the schedule and the higher impact change requests should be tackled first." + elif self.goal_type == "tight": + task_info = "Edit the schedule of the change requests by setting the start and end dates so that the change requests do not overlap. There should not be more than one hour between conescutive change requests in the schedule." + elif self.goal_type == "tight priority": + task_info = "Edit the schedule of the change requests by setting the start and end dates so that the change requests do not overlap. There should not be more than one hour between conescutive change requests in the schedule and the higher impact change requests should be tackled first." + + task_info += " Finally, all change requests must respect the desired durations, which are determined by the risk level:\n" + task_info += " - High risk: 3 days \n" + task_info += " - Moderate risk: 2 days \n" + task_info += " - Low risk: 1 day \n" + + return task_info + + +class EditIncidentTask(EditRecordTask): + """ + Task to edit a new incident in the system. + + """ + + expected_fields_path = EXPECTED_INCIDENT_FORM_FIELDS_PATH + + def __init__( + self, + instance=None, + fixed_config: dict = None, + new_values: dict = None, + record_sys_id: str = None, + **kwargs, + ) -> None: + super().__init__( + instance=instance, + form_url="/now/nav/ui/classic/params/target/incident.do", + table_label="incident", + prohibited_fields=["state"], + fixed_config=fixed_config, + new_values=new_values, + record_sys_id=record_sys_id, + ) + if self.new_values is None: + self.new_values = {"assigned_to": "fred.luddy"} + self.__dict__.update(kwargs) + + +class CreateItemRequestTask(GenericNewRecordTask, CompositionalBuildingBlockTask): + """ + Task to create a new item request in the system. + """ + + expected_fields_path = EXPECTED_REQUEST_ITEM_FORM_FIELDS_PATH + + def __init__( + self, instance=None, fixed_config: dict = None, check_record_created=True, **kwargs + ) -> None: + super().__init__( + instance=instance, + form_url="/now/nav/ui/classic/params/target/sc_req_item.do", + table_label="sc_req_item", + fixed_config=fixed_config, + check_record_created=check_record_created, + ) + self.__dict__.update(kwargs) + +local_vars = locals().copy() __TASKS__ = [ var - for var in locals().values() - if isinstance(var, type) - and issubclass(var, GenericNewRecordTask) + for var in local_vars.values() + if inspect.isclass(var) + and not issubclass(var, CompositionalBuildingBlockTask) + and issubclass(var, ServiceNowFormTask) and var is not GenericNewRecordTask + and var is not ServiceNowFormTask ] diff --git a/src/browsergym/workarena/tasks/knowledge.py b/src/browsergym/workarena/tasks/knowledge.py index fc482c6..85da3c9 100644 --- a/src/browsergym/workarena/tasks/knowledge.py +++ b/src/browsergym/workarena/tasks/knowledge.py @@ -5,15 +5,20 @@ import json import logging +import re from playwright.sync_api import Page +from typing import Tuple from urllib import parse from .base import AbstractServiceNowTask -from ..config import KB_NAME, KB_FILEPATH, KB_CONFIG_PATH, SNOW_BROWSER_TIMEOUT +from .comp_building_block import CompositionalBuildingBlockTask +from .utils.utils import check_url_suffix_match + +from ..api.utils import table_api_call +from ..config import KB_FILEPATH, KB_CONFIG_PATH, KB_NAME, SNOW_BROWSER_TIMEOUT from ..install import check_knowledge_base from ..instance import SNowInstance -from .utils.utils import check_url_suffix_match class KnowledgeBaseSearchTask(AbstractServiceNowTask): @@ -32,7 +37,33 @@ class KnowledgeBaseSearchTask(AbstractServiceNowTask): """ - def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: + def __init__( + self, + instance=None, + fixed_config: dict = None, + is_correct: bool = True, + is_only_navigating: bool = False, + search_by_title: bool = False, + seed: int = None, + **kwargs, + ) -> None: + """ + Parameters: + ----------- + instance: SNowInstance + The ServiceNow instance to run the task on. + fixed_config: + A fixed configuration for the task, if required. + is_correct: bool + Used for the compositional task. + If false, the answer is highlighted in 'red' instead of 'yellow' when using cheat. + is_only_navigating: bool + Used for the compositional task. + If we only are navigating and not searching, change the goal for the agent. + search_by_title: bool + Used for the compositional task. + If true, clicks on the article title using the article name, else opens the first article. + """ super().__init__( seed=seed, instance=instance, @@ -42,9 +73,18 @@ def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: # Load the knowledge base and check its integrity with open(KB_FILEPATH, "r") as f: self.kb_entries = json.load(f) - _, requires_install, requires_delete = check_knowledge_base( - self.instance, kb_data=self.kb_entries, kb_name=KB_NAME - ) + if hasattr(self, "_base_initial_instance"): + _, requires_install, requires_delete = check_knowledge_base( + self._base_initial_instance, # if user does not have permission to view the kb then this breaks + kb_name=KB_NAME, + kb_data=self.kb_entries, # Need admin permissions to check + ) + else: + _, requires_install, requires_delete = check_knowledge_base( + SNowInstance(), # instance would be the non-admin instance here and this might break in case user does not have required permissions + kb_name=KB_NAME, + kb_data=self.kb_entries, # Need admin permissions to check + ) with open(KB_CONFIG_PATH, "r") as f: self.all_configs = json.load(f) if any([requires_install, requires_delete]): @@ -53,6 +93,14 @@ def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: "See README for setup instructions." ) self.fixed_config = fixed_config + self.config = None + + # Attributes for compositional task + self.is_correct = is_correct + self.is_only_navigating = is_only_navigating + self.search_by_title = search_by_title + + self.__dict__.update(kwargs) def _wait_for_ready(self, page: Page) -> None: """ @@ -92,13 +140,33 @@ def setup_goal(self, page: Page) -> tuple[str, dict]: self.answer = config["value"] self.alternative_answers = config["alternative_answers"] self.question = config["question"] + if self.search_by_title: + self.kb_article_title = config["kb_article_title"] # Generate goal - goal = f'Answer the following question using the knowledge base: "{self.question}"' + if self.is_only_navigating: + goal = f'Navigate to a relevant article in the knowledge base by searching for: "{self.item}" and open the article: "{self.kb_article_title}"' + else: + goal = f'Answer the following question using the knowledge base: "{self.question}"' info = {} return goal, info + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + class_name = self.__class__.__name__ + class_name = class_name.replace("Task", "") + # Split the words + words = re.findall(r"[A-Z][^A-Z]*", class_name) + class_name_formatted = " ".join(words) + + task_info = f"- {class_name_formatted}: {self.item} \n" + + return task_info + def cheat(self, page: Page, chat_messages: list[str]) -> None: super().cheat(page=page, chat_messages=chat_messages) self._wait_for_ready(page) @@ -112,7 +180,10 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: # Click on the article with page.expect_navigation(): - iframe.locator("a.kb-title").first.click() + if self.search_by_title: + iframe.locator(f'a.kb-title:has-text("{self.kb_article_title}")').click() + else: + iframe.locator("a.kb-title").first.click() # Color the query and answer (this is just for visualization, it changes nothing to the validation) paragraphs = iframe.locator("p") @@ -125,10 +196,16 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: self.item, f'{self.item}', ) - inner_html = inner_html.replace( - str(self.answer), - f'{self.answer}', - ) + if self.is_correct: + inner_html = inner_html.replace( + str(self.answer), + f'{self.answer}', + ) + else: + inner_html = inner_html.replace( + str(self.answer), + f'{self.answer}', + ) paragraph.evaluate(f"element => element.innerHTML = `{inner_html}`") break @@ -137,16 +214,6 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: chat_messages.append({"role": "assistant", "message": str(self.answer)}) def validate(self, page: Page, chat_messages: list[str]) -> tuple[float, bool, str, dict]: - right_url = check_url_suffix_match(page, expected_url="target/kb", task=self) - if not right_url: - return ( - 0, - False, - "", - { - "message": f"The page is not in the right URL to validate task {self.__class__.__name__}." - }, - ) if chat_messages and chat_messages[-1]["role"] == "assistant": answer = chat_messages[-1]["message"] else: @@ -160,9 +227,133 @@ def validate(self, page: Page, chat_messages: list[str]) -> tuple[float, bool, s accepted_answers = [a.lower() for a in [self.answer] + self.alternative_answers] answer = answer.lower() if any(a in answer for a in accepted_answers): - return 1, True, "That is correct, thank you!", {"message": "Correct answer."} + return ( + 1, + True, + "That is correct, thank you!", + {"message": "Correct answer."}, + ) else: - return 0, False, "", {"message": "Incorrect answer provided by the assistant."} + return ( + 0, + False, + "", + {"message": "Incorrect answer provided by the assistant."}, + ) + + +class AddCommentToKnowledgeArticleTask(AbstractServiceNowTask, CompositionalBuildingBlockTask): + """ + Task to add a comment to a knowledge base article. Only used as a part of the compositional task for edit knowledge base + Parameters: + ----------- + instance: SNowInstance + The instance on which to create the record. + fixed_config: dict + Configuration to use for the task. + """ + + def __init__( + self, seed: int = None, instance=None, fixed_config: dict = None, **kwargs + ) -> None: + super().__init__( + seed=seed, + instance=instance, + start_rel_url="/now/nav/ui/classic/params/target/kb?id=kb_home", + user_roles=[], + ) + self.fixed_config = fixed_config + if self.fixed_config is None: + raise Exception("Please provide a config for the add comment task.") + self.__dict__.update(kwargs) + + def setup_goal(self, page: Page) -> tuple[str, dict]: + super().setup_goal(page=page) + config = self.fixed_config + + if "kb_article_title" not in config.keys(): + raise Exception("Need title in config file...") + self.article_name = config["kb_article_title"] + adhoc_kb_response = table_api_call( + instance=self.instance, # admin permissions to contribute to the KB + table="kb_knowledge", + method="GET", + params={ + "sysparm_query": f"short_description={self.article_name}", + }, + )["result"] + if len(adhoc_kb_response) != 1: + raise Exception("Required article not found, please fix config...") + + self.kb_article_sys_id = adhoc_kb_response[0]["sys_id"] + self.comment = config["comment"] + + goal = f'Add the following comment to the knowledge base: "{self.comment}"' + info = {} + + return goal, info + + def _wait_for_ready(self, page: Page) -> None: + """ + Checks that the main iframe is fully loaded + + """ + # TODO: We don't use the flag-based method used in other tasks + # because gsft_main doesn't have the event we register + # on this page. Not sure why. + logging.debug(f"Waiting for page to be fully loaded") + page.wait_for_load_state("networkidle") + page.wait_for_selector('iframe[name="gsft_main"]') + logging.debug(f"Detected page ready") + + # Get main iframe + # XXX: We use a loop because sometimes the iframe evaluates to None + # even though we wait for it to be ready. This seems like a + # playwright bug. + timeout = SNOW_BROWSER_TIMEOUT + while timeout > 0: + iframe = page.frame(name="gsft_main") + if iframe: + break + page.wait_for_timeout(100) + timeout -= 100 + else: + raise TimeoutError( + f"Timed out waiting for iframe to be ready in {self.instance.snow_url}" + ) + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + class_name = self.__class__.__name__ + class_name = class_name.replace("Task", "") + # Split the words + words = re.findall(r"[A-Z][^A-Z]*", class_name) + class_name_formatted = " ".join(words) + + task_info = f"- {class_name_formatted}: Add the comment '{self.comment}' for the article with title {self.article_name} \n" + + return task_info + + def cheat(self, page: Page, chat_messages: list[str]) -> None: + super().cheat(page=page, chat_messages=chat_messages) + + # Check if we need to do something else, gsft_main is not loading, it seems to load when navigating from the search, so might need for compositional tasks + self._wait_for_ready(page) + frame = page.frame("gsft_main") + frame.locator("button.comment-text").click() + frame.frame_locator('iframe[title="Rich Text Area"]').locator("html").click() + frame.frame_locator('iframe[title="Rich Text Area"]').get_by_label("Comments").fill( + self.comment + ) + frame.get_by_role("button", name="Submit").click() + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + return super().validate(page, chat_messages) -__TASKS__ = [KnowledgeBaseSearchTask] +__TASKS__ = [ + KnowledgeBaseSearchTask, +] diff --git a/src/browsergym/workarena/tasks/list.py b/src/browsergym/workarena/tasks/list.py index c60ce9a..6153f7f 100644 --- a/src/browsergym/workarena/tasks/list.py +++ b/src/browsergym/workarena/tasks/list.py @@ -15,6 +15,8 @@ from urllib import parse from warnings import warn +from .comp_building_block import CompositionalBuildingBlockTask + from ..api.utils import table_api_call, table_column_info from ..config import ( SNOW_BROWSER_TIMEOUT, @@ -35,6 +37,7 @@ EXPECTED_CHANGE_REQUEST_COLUMNS_PATH, EXPECTED_HARDWARE_COLUMNS_PATH, EXPECTED_INCIDENT_COLUMNS_PATH, + EXPECTED_PROBLEM_COLUMNS_PATH, EXPECTED_SERVICE_CATALOG_COLUMNS_PATH, EXPECTED_USER_COLUMNS_PATH, ) @@ -76,8 +79,34 @@ }, } +EXTRACT_USER_LIST_INFO_CONFIG = [ + { + "start_rel_url": "/now/nav/ui/classic/params/target/sys_user_list.do%3Fsysparm_query%3Dactive%253Dtrue%255Ecompany%253D81fd65ecac1d55eb42a426568fc87a63%255Eemail%253Dlucius.bagnoli%40example.com%26sysparm_first_row%3D1%26sysparm_view%3D", + "fields": { + "user_name": "User ID", + "email": "Email", + "first_name": "First name", + "last_name": "Last name", + }, + "expected_values": [ + { + "user_name": "lucius.bagnoli", + "email": "lucius.bagnoli@example.com", + "first_name": "Lucius", + "last_name": "Bagnoli", + } + ], + } +] + class ServiceNowListTask(AbstractServiceNowTask): + + @classmethod + def all_configs(cls) -> List[dict]: + with open(cls.config_path, "r") as f: + return json.load(f) + def get_init_scripts(self) -> List[str]: return super().get_init_scripts() + ["registerGsftMainLoaded();"] @@ -129,12 +158,10 @@ def _extract_list_info(self, page: Page, with_data=False): } # Get column info - fields = list_info["fields"].split(",") list_info["columns"] = table_column_info( instance=self.instance, table=list_info["glide_table"], ) - list_info["columns"] = {k: v for k, v in list_info["columns"].items() if k in fields} # Get the list data if with_data: @@ -189,43 +216,46 @@ class SortListTask(ServiceNowListTask): Configuration to use for the task. If provided, the task will use the provided configuration instead of selecting a random one. See browsergym/workarena/data_files/task_configs/sort_change_request_list_task.json for an example of a configuration file. - config_path: - The path to the JSON file containing all configurations for the task. Provided by subclasses expected_fields_path: The path to the JSON file containing all expected fields for the task. Provided by subclasses """ def __init__( self, - seed: int, + seed: int = None, instance=None, list_url="", forbidden_fields=[], fixed_config: dict = None, - config_path: str = None, expected_fields_path: str = None, + **kwargs, ) -> None: super().__init__(seed=seed, instance=instance, start_rel_url=list_url) self.min_sort_len = 1 self.max_sort_len = 3 self.forbidden_fields = forbidden_fields self.fixed_config = fixed_config - if config_path: - with open(config_path, "r") as f: - self.all_configs = json.load(f) + self.config = None + if hasattr(self, "config_path"): + self.all_configs = self.all_configs() + with open(expected_fields_path, "r") as f: self.expected_fields = set(json.load(f)) + self.list_info = None + self.__dict__.update(kwargs) def setup_goal(self, page: Page) -> tuple[str, dict]: super().setup_goal(page=page) # Get the task configuration - config = self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) - self.sort_fields = config["sort_fields"] - self.sort_dirs = config["sort_dirs"] + self.config = ( + self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) + ) + self.sort_fields = self.config["sort_fields"] + self.sort_dirs = self.config["sort_dirs"] # Get the task goal - goal = config["goal"] + goal = self.config["goal"] info = {} return goal, info @@ -236,10 +266,13 @@ def start(self, page: Page) -> None: # Ensure that the fields that need to be sorted are visible (task feasibility check) self.list_info = self._extract_list_info(page) - visible_columns = set(self.list_info["fields"].split(",")) - assert ( - set(self.sort_fields) <= visible_columns and visible_columns == self.expected_fields - ), f"Fields {self.sort_fields} are not all visible in the list. Re-run workarena-install to correct this." + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + return self.config["goal"] + "\n" def _generate_all_configs(self, seed: int, page: Page, n_fields_to_sort: int): self.setup(seed=seed, page=page) @@ -311,7 +344,7 @@ def _generate_random_config(self, page: Page): sort_dirs_txt = [dir_txt[sort_dir] for sort_dir in self.sort_dirs] # check if the task is already solved (can happen if the chosen field is already sorted in the default view) - _, done, _, _ = self.validate(self.page, []) + _, done, _, _ = self.validate(page, []) # if so, pick new fields if done: logging.warning("Trivial config for sort list task, picking a new config.") @@ -329,6 +362,8 @@ def _generate_random_config(self, page: Page): def cheat(self, page: Page, chat_messages: list[str]) -> None: super().cheat(page=page, chat_messages=chat_messages) self._wait_for_ready(page) + if self.list_info is None: + self.list_info = self._extract_list_info(page) iframe, _, _ = self._get_visible_list(page) @@ -408,8 +443,8 @@ def validate( # ... retrieve list list_info = self._extract_list_info(page) # ... get sorting info - sort_by = self.page.evaluate(f'{list_info["js_selector"]}.getOrderBy()') - sort_dir = self.page.evaluate(f'{list_info["js_selector"]}.sortDir') + sort_by = page.evaluate(f'{list_info["js_selector"]}.getOrderBy()') + sort_dir = page.evaluate(f'{list_info["js_selector"]}.sortDir') # ... check if the list is sorted correctly if sort_by == self.sort_fields[0] and sort_dir.lower() == self.sort_dirs[0]: return ( @@ -469,30 +504,31 @@ class FilterListTask(ServiceNowListTask): Configuration to use for the task. If provided, the task will use the provided configuration instead of selecting a random one. See browsergym/workarena/data_files/task_configs/filter_change_request_list_task.json for an example of a configuration file. - config_path: - The path to the JSON file containing all configurations for the task. Provided by subclasses expected_fields_path: The path to the JSON file containing all expected fields for the task. Provided by subclasses """ def __init__( self, - seed: int, + seed: int = None, instance=None, list_url="", fixed_config: dict = None, - config_path: str = None, expected_fields_path: str = None, + **kwargs, ) -> None: self.min_filter_len = 2 self.max_filter_len = 5 super().__init__(seed=seed, instance=instance, start_rel_url=list_url) self.fixed_config = fixed_config - if config_path: - with open(config_path, "r") as f: - self.all_configs = json.load(f) + self.config = None + if hasattr(self, "config_path"): + self.all_configs = self.all_configs() + with open(expected_fields_path, "r") as f: self.expected_fields = set(json.load(f)) + self.table_name = list_url.split("/")[-1].split("_list.do")[0] + self.__dict__.update(kwargs) def setup_goal(self, page: Page) -> tuple[str, dict]: super().setup_goal(page=page) @@ -501,36 +537,48 @@ def setup_goal(self, page: Page) -> tuple[str, dict]: config = self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) self.filter_columns = config["filter_columns"] self.filter_values = config["filter_values"] + # Base filter configs do not have filter_operands, so we default to "is" + self.filter_operators = config.get("filter_operators", ["is" for _ in self.filter_columns]) self.filter_kind = config["filter_kind"] - self.list_info = config["list_info"] + list_info = config.get("list_info") + if list_info is None: + list_info = {"columns": table_column_info(self.instance, self.table_name)} + self.list_info = list_info self.filter_len = len(self.filter_columns) # Generate goal - goal = ( - f"Create a filter for the list to extract all entries where " - + f" {'and' if self.filter_kind == 'AND' else 'or'} ".join( - [ - f'"{self.list_info["columns"][col]["label"]}" is "{val}"' - for col, val in zip(self.filter_columns, self.filter_values) - ] - ) - + "." - ) + goal = self.get_pretty_printed_description(goal=True) info = {} return goal, info def start(self, page: Page) -> None: super().start(page) - self._wait_for_ready(page) - # Assert that required fields are visible (task feasibility check) - visible_list_info = self._extract_list_info(page) - visible_columns = set(visible_list_info["fields"].split(",")) - assert ( - set(self.filter_columns) <= visible_columns and visible_columns == self.expected_fields - ), f"Fields {self.filter_columns} are not all visible in the list. Re-run workarena-install to correct this." + def get_pretty_printed_description(self, goal=False) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + + args: + goal: bool + If True, return as the goal of the task (without the starting dash) + """ + task_info = "" if goal else "- " + task_info += ( + f"Create a filter for the list to extract all entries where:" + + f" {'and' if self.filter_kind == 'AND' else 'or'} ".join( + [ + f'\n - "{self.list_info["columns"][col]["label"]}" {filter_operator} "{val}"' + for col, filter_operator, val in zip( + self.filter_columns, self.filter_operators, self.filter_values + ) + ] + ) + ) + + return task_info def _generate_random_config(self, page: Page): self.setup(page=page) @@ -563,7 +611,7 @@ def _generate_random_config(self, page: Page): # We do this by loading a single record at random and using its values # This is significantly faster than loading all records and then filtering offset = self.random.randint( - 0, self.page.evaluate(f'{self.list_info["js_selector"]}.grandTotalRows') + 0, page.evaluate(f'{self.list_info["js_selector"]}.grandTotalRows') ) data = table_api_call( instance=self.instance, @@ -652,7 +700,7 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: f'.filterToolbar .filerTableAction:text-is("{self.filter_kind}")' ).click() # TODO: Hack to solve bug where the filter condition has not yet appeared - self.page.wait_for_timeout(1000) + page.wait_for_timeout(1000) # Refresh since new rows are added at each iteration filter_rows = iframe.locator(".filter_row") @@ -664,8 +712,14 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: field_selector.select_option(self.filter_columns[i]) # Select the right operator - logging.debug("Choosing operator =") - row.locator("select.condOperator").select_option("=") + operator = self.filter_operators[i] + operator_symbol = ( + row.locator("select.condOperator") + .get_by_text(operator, exact=True) + .get_attribute("value") + ) + logging.debug(f"Choosing operator {operator}") + row.locator("select.condOperator").select_option(operator_symbol) # Fill in the value logging.debug("Filling in value " + self.filter_values[i]) @@ -718,26 +772,19 @@ def validate( }, ) self._wait_for_ready(page) - if self.filter_kind not in ["AND", "OR"]: - raise NotImplementedError("Only AND and OR filters are supported.") - # Excludes AND because that's the default and its sep is ^ which matches everywhere - query_sep = {"OR": "^NQ"} - # Retrieve list + # Retrieve the current query list_info = self._extract_list_info(page) - - # Check if the list is filtered correctly current_query = list_info["query"] + # Replace "new query" statements with the standard OR separator + current_query = current_query.replace("^NQ", "^OR") + # Validate query kind is ok - current_kind = None - for kind in query_sep: - if query_sep[kind] in current_query: - current_kind = kind - current_sep = query_sep[kind] - break + if "^OR" in current_query: + current_kind = "OR" + current_sep = "^OR" else: - # If no separator is found, then the query is just assumed to be AND (it's a single condition) current_kind = "AND" current_sep = "^" @@ -760,24 +807,44 @@ def validate( # This is the tricky part because we need to expand the values to their display values # We also need to handle the case where the value is a reference current_values = [x.split("=")[1] for x in current_query] - for col, val in zip(current_columns, current_values): - col_info = self.list_info["columns"][col] + # Handle filtering across multiple rows + if len(set(current_columns)) < len(current_columns): + if len(set(current_columns)) != 1: + raise Exception("Filtering is only allowed across rows for the same column.") + # Filter multiple rows with a column + is_homogenous_filter = True + else: + # Current setting where we use multiple columns to filter + is_homogenous_filter = False + for index, (col, val) in enumerate(zip(current_columns, current_values)): + col_info = self.list_info["columns"][col] # Get the column type if col_info["type"] == "reference" and val != "": # Get the reference table ref_table = col_info["reference"] ref_field = col_info["reference_attributes"]["display_field"] - # Get the reference display value - current_values[current_columns.index(col)] = table_api_call( - instance=self.instance, - table=ref_table, - params={ - "sysparm_query": f"sys_id={val}", - "sysparm_fields": ref_field, - "sysparm_display_value": "all", - }, - )["result"][0][ref_field]["display_value"] + if is_homogenous_filter: + current_values[index] = table_api_call( + instance=self.instance, + table=ref_table, + params={ + "sysparm_query": f"sys_id={val}", + "sysparm_fields": ref_field, + "sysparm_display_value": "all", + }, + )["result"][0][ref_field]["display_value"] + else: + # Get the reference display value + current_values[current_columns.index(col)] = table_api_call( + instance=self.instance, + table=ref_table, + params={ + "sysparm_query": f"sys_id={val}", + "sysparm_fields": ref_field, + "sysparm_display_value": "all", + }, + )["result"][0][ref_field]["display_value"] elif col_info["type"] == "choice": # Get the choice display value @@ -794,114 +861,409 @@ def validate( return 1, True, "Nice work, thank you!", {"message": "Correct filter."} +class ExtractListInfoTask(ServiceNowListTask): + """ + Extract information from some fields in a list. Works with any list. + + Parameters: + ----------- + instance: SNowInstance + The instance to use. + list_url: str + The relative URL of the list to filter. + fixed_config: dict + Configuration to use for the task. If provided, the task will use the provided configuration instead of + selecting a random one. See browsergym/workarena/data_files/task_configs/filter_change_request_list_task.json + for an example of a configuration file. + config_path: + The path to the JSON file containing all configurations for the task. Provided by subclasses + list_name: str + Name of the list to extract information from. + list_url: str + url of the list to extract information from. + unique_field_name: str + Name of the field used as unique in the list. This field is required in configs. + """ + + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + configs: str = "", + list_name: str = "", + list_url: str = "", + unique_field_name: str = "", + **kwargs, + ) -> None: + super().__init__( + seed=seed, instance=instance, start_rel_url=list_url + ) # For these tasks, the start URL is defined in the setup method, as the URL depends on the configuration + self.fixed_config = fixed_config + self.config = None + self.all_configs = configs + self.list_name = list_name + self.table_name = "" + self.unique_field_name = unique_field_name + self.__dict__.update(kwargs) + + def setup_goal(self, page: Page) -> tuple[str, dict]: + super().setup_goal(page=page) + + # Get the task configuration + config = self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) + self.fields = config["fields"] # mapping between fields and their display names + self.printed_field_names = { + v: k for k, v in self.fields.items() + } # mapping between fields and their system names + self.expected_values = config[ + "expected_values" + ] # mapping between fields and their expected values + # This is setup here because the start_url depends on the config + assert ( + self.unique_field_name in self.fields.keys() + ), f"Unique field name {self.unique_field_name} not in fields." + assert all( + [self.unique_field_name in expected_value for expected_value in self.expected_values] + ), f"Unique field name {self.unique_field_name} not in expected values." + + if not self.start_url or self.start_url == self.instance.snow_url: + self.start_rel_url = config["start_rel_url"] + self.start_url = self.instance.snow_url + self.start_rel_url + # table_name can be passed in the constructor or extracted from the start_rel_url, located in the config + if self.table_name is None: + self.table_name = self.start_rel_url.split("/")[-1].split("_list.do")[0] + + goal = self.get_pretty_printed_description() + info = {} + + return goal, info + + def start(self, page: Page) -> None: + super().start(page) + # TODO: We should add a check to make sure the required columns are present in the list + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks and used as goal in L1 tasks. + called by subclasses + """ + print_field_names = list(self.fields.values()) + print_field_names.remove( + self.fields[self.unique_field_name] + ) # the unique fields are the keys in the dict + if len(print_field_names) > 1: + fields_str = ( + '"' + '", "'.join(print_field_names[:-1]) + f'" and "{print_field_names[-1]}"' + ) + printed_unique_field_name = self.fields[self.unique_field_name] + task_description = ( + f"- Extract information of field(s) {fields_str} " + + f'from the "{self.list_name}" list. Return the result as a json where keys are the values of the "{printed_unique_field_name}" field and values are mappings between the fields and the extracted information. Please provide this information in the chat.' + ) + else: + fields_str = print_field_names[0] + task_description = f'- Extract information of field "{fields_str}" from the "{self.list_name}" list. Please provide this information in the chat.' + + return task_description + + def _wait_for_ready(self, page: Page) -> bool: + """ + Waits for the main iframe to be fully loaded; over-rides the parent method as the cheat + can be called on a filtered list on which there is no gsft_main. + + Returns True if the gsft_main is present, False otherwise. + """ + gsft_main_present = False + logging.debug(f"Waiting up to 3 seconds for gsft_main to be ready") + try: + page.wait_for_function( + "typeof window.gsft_main !== 'undefined' && window.gsft_main.WORKARENA_LOAD_COMPLETE", + timeout=3000, + ) + logging.debug("Detected gsft_main ready") + gsft_main_present = True + except TimeoutError: + logging.debug( + "Timed out waiting for gsft_main to be ready; searching for GlideList API directly" + ) + pass + + logging.debug("Waiting for Glide list API to be available") + if gsft_main_present: + page.wait_for_function("window.gsft_main.GlideList2 !== undefined") + else: + page.wait_for_function("window.GlideList2 !== undefined") + + logging.debug("Detected Glide list API ready") + + return gsft_main_present + + def cheat(self, page: Page, chat_messages: list[str]) -> None: + super().cheat(page=page, chat_messages=chat_messages) + right_url = check_url_suffix_match(page, expected_url=self.start_url, task=self) + if not right_url: + return + gft_main_present = self._wait_for_ready(page) + if gft_main_present: + main_element = page.wait_for_selector("iframe#gsft_main").content_frame() + else: + main_element = page + + main_element.wait_for_selector( + f"#hdr_{self.table_name}" + ) # Selector for the name of the columns + # system name mapped to their order in the table + all_column_elements = main_element.query_selector_all(f"#hdr_{self.table_name} th") + required_fields_order = {} + for i, element in enumerate(all_column_elements): + if element.get_attribute("name") in self.fields: + required_fields_order[element.get_attribute("name")] = i + + # Lines of the table + table_lines = main_element.query_selector_all( + f".list2_body [record_class={self.table_name}]" + ) + + # will hold the values to extract + table_values = {} + + # Extract the values of the required fields + for line_element in table_lines: + line_fields = line_element.query_selector_all("td") + line_values = {} + for field, order in required_fields_order.items(): + printed_field_name = self.fields[field] + line_values[printed_field_name] = line_fields[order].inner_text() + printed_unique_value_name = self.fields[self.unique_field_name] + unique_field_value = line_values[printed_unique_value_name] + line_values.pop(printed_unique_value_name) + table_values[unique_field_value] = line_values + + # Add the "extracted" answer to the chat messages + if len(self.fields) > 2: + chat_messages.append({"role": "assistant", "message": json.dumps(table_values)}) + # In this case, we expect only one field to be extracted + else: + expected_field = list(self.fields.keys() - {self.unique_field_name})[0] + pretty_field_name = self.fields[expected_field] + # Here we assume that unique_field_value is unique in the table_values + chat_messages.append( + { + "role": "assistant", + "message": str(table_values[unique_field_value][pretty_field_name]), + } + ) + + def validate( + self, page: playwright.sync_api.Page, chat_messages: list[str] + ) -> Tuple[float, bool, str, dict]: + """ + Validate the solution + + Note: current implementation is limited to AND and OR filters (single type per filter) with equality operators + + """ + if ( + len(chat_messages) == 0 + or chat_messages[-1]["role"] != "assistant" + or not chat_messages[-1]["message"] + ): + return 0, False, "", {"message": "No extracted values found."} + + # When 2 or more fields (unique field is always present so at least 2 fields are present), we expect a dict + # Otherwise, we only look for the presence of the expected value in the message sent by the agent + if len(self.fields) > 2: + answer = json.loads(chat_messages[-1]["message"]) + for expected_line in self.expected_values: + # Check if the line is in the visible lines + if expected_line[self.unique_field_name] not in answer: + return ( + 0, + False, + "", + { + "message": f"Value {expected_line[self.unique_field_name]} for unique field {self.unique_field_name} not found in the list." + }, + ) + # Check if the values are correct + unique_value = expected_line[self.unique_field_name] + # This checks all fields inside the dict for the unique value + for field, value in expected_line.items(): + # The unique field's presence is implicitly validated by the above check + if field == self.unique_field_name: + continue + printed_field_name = self.fields[field] + if answer[unique_value][printed_field_name] != value: + return 0, False, "", {"message": "Incorrect value."} + # In this case, we expect only one field to be extracted + else: + # get the field that is not the unique field + field = list(self.fields.keys() - {self.unique_field_name})[0] + expected_value = str(self.expected_values[0][field]) + if expected_value not in chat_messages[-1]["message"]: + return 0, False, "", {"message": "Incorrect value."} + + return ( + 1, + True, + "Nice work, thank you!", + {"message": "Correct information extracted."}, + ) + + class FilterAssetListTask(FilterListTask): + config_path = FILTER_ASSET_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, instance=instance, list_url=LISTS["alm_asset"]["url"], fixed_config=fixed_config, - config_path=FILTER_ASSET_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_ASSET_LIST_COLUMNS_PATH, + **kwargs, ) class FilterChangeRequestListTask(FilterListTask): + config_path = FILTER_CHANGE_REQUEST_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, instance=instance, list_url=LISTS["change_request"]["url"], fixed_config=fixed_config, - config_path=FILTER_CHANGE_REQUEST_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_CHANGE_REQUEST_COLUMNS_PATH, + **kwargs, ) class FilterHardwareListTask(FilterListTask): + config_path = FILTER_HARDWARE_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, instance=instance, list_url=LISTS["alm_hardware"]["url"], fixed_config=fixed_config, - config_path=FILTER_HARDWARE_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_HARDWARE_COLUMNS_PATH, + **kwargs, ) class FilterIncidentListTask(FilterListTask): + config_path = FILTER_INCIDENT_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, instance=instance, list_url=LISTS["incident"]["url"], fixed_config=fixed_config, - config_path=FILTER_INCIDENT_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_INCIDENT_COLUMNS_PATH, + **kwargs, ) +class FilterProblemListForWorkLoadBalancingTask(FilterListTask, CompositionalBuildingBlockTask): + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + **kwargs, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + list_url="/now/nav/ui/classic/params/target/problem_list.do", + fixed_config=fixed_config, + expected_fields_path=EXPECTED_PROBLEM_COLUMNS_PATH, + **kwargs, + ) + + def get_pretty_printed_description(self, goal=False) -> str: + """Override the parent method to provide a more detailed description of the task""" + + return self.goal + + class FilterServiceCatalogItemListTask(FilterListTask): + config_path = FILTER_SERVICE_CATALOG_ITEM_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, instance=instance, list_url=LISTS["sc_cat_item"]["url"], fixed_config=fixed_config, - config_path=FILTER_SERVICE_CATALOG_ITEM_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_SERVICE_CATALOG_COLUMNS_PATH, + **kwargs, ) class FilterUserListTask(FilterListTask): + config_path = FILTER_USER_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, instance=instance, list_url=LISTS["sys_user"]["url"], fixed_config=fixed_config, - config_path=FILTER_USER_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_USER_COLUMNS_PATH, + **kwargs, ) class SortAssetListTask(SortListTask): + config_path = SORT_ASSET_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, @@ -909,17 +1271,20 @@ def __init__( list_url=LISTS["alm_asset"]["url"], forbidden_fields=LISTS["alm_asset"]["forbidden_fields"], fixed_config=fixed_config, - config_path=SORT_ASSET_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_ASSET_LIST_COLUMNS_PATH, + **kwargs, ) class SortChangeRequestListTask(SortListTask): + config_path = SORT_CHANGE_REQUEST_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, @@ -927,17 +1292,20 @@ def __init__( list_url=LISTS["change_request"]["url"], forbidden_fields=LISTS["change_request"]["forbidden_fields"], fixed_config=fixed_config, - config_path=SORT_CHANGE_REQUEST_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_CHANGE_REQUEST_COLUMNS_PATH, + **kwargs, ) class SortHardwareListTask(SortListTask): + config_path = SORT_HARDWARE_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, @@ -945,17 +1313,20 @@ def __init__( list_url=LISTS["alm_hardware"]["url"], forbidden_fields=LISTS["alm_hardware"]["forbidden_fields"], fixed_config=fixed_config, - config_path=SORT_HARDWARE_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_HARDWARE_COLUMNS_PATH, + **kwargs, ) class SortIncidentListTask(SortListTask): + config_path = SORT_INCIDENT_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, @@ -963,17 +1334,20 @@ def __init__( list_url=LISTS["incident"]["url"], forbidden_fields=LISTS["incident"]["forbidden_fields"], fixed_config=fixed_config, - config_path=SORT_INCIDENT_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_INCIDENT_COLUMNS_PATH, + **kwargs, ) class SortServiceCatalogItemListTask(SortListTask): + config_path = SORT_SERVICE_CATALOG_ITEM_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, @@ -981,17 +1355,20 @@ def __init__( list_url=LISTS["sc_cat_item"]["url"], forbidden_fields=LISTS["sc_cat_item"]["forbidden_fields"], fixed_config=fixed_config, - config_path=SORT_SERVICE_CATALOG_ITEM_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_SERVICE_CATALOG_COLUMNS_PATH, + **kwargs, ) class SortUserListTask(SortListTask): + config_path = SORT_USER_LIST_CONFIG_PATH + def __init__( self, - seed: int, + seed: int = None, instance=None, fixed_config: dict = None, + **kwargs, ) -> None: super().__init__( seed=seed, @@ -999,12 +1376,52 @@ def __init__( list_url=LISTS["sys_user"]["url"], forbidden_fields=LISTS["sys_user"]["forbidden_fields"], fixed_config=fixed_config, - config_path=SORT_USER_LIST_CONFIG_PATH, expected_fields_path=EXPECTED_USER_COLUMNS_PATH, + **kwargs, + ) + + +class ExtractUserListInfoTask(ExtractListInfoTask, CompositionalBuildingBlockTask): + def __init__( + self, + seed: int = None, + instance=None, + fixed_config: dict = None, + config_path=EXTRACT_USER_LIST_INFO_CONFIG, + list_name="User", + unique_field_name="user_name", + **kwargs, + ) -> None: + super().__init__( + seed=seed, + instance=instance, + fixed_config=fixed_config, + config_path=config_path, + list_name=list_name, + unique_field_name=unique_field_name, + table_name="sys_user", + **kwargs, ) # Register all tasks -__TASKS__ = [ - value for name, value in locals().items() if re.compile(r"^Filter\w+ListTask$").match(name) -] + [value for name, value in locals().items() if re.compile(r"^Sort\w+ListTask$").match(name)] +__TASKS__ = ( + [ + value + for name, value in locals().items() + if re.compile(r"^Filter\w+ListTask$").match(name) + and not issubclass(value, CompositionalBuildingBlockTask) + ] + + [ + value + for name, value in locals().items() + if re.compile(r"^Sort\w+ListTask$").match(name) + and not issubclass(value, CompositionalBuildingBlockTask) + ] + + [ + value + for name, value in locals().items() + if re.compile(r"^Extract\w+ListInfoTask$").match(name) + and not issubclass(value, CompositionalBuildingBlockTask) + ] +) diff --git a/src/browsergym/workarena/tasks/mark_duplicate_problem.py b/src/browsergym/workarena/tasks/mark_duplicate_problem.py new file mode 100644 index 0000000..f6a55e6 --- /dev/null +++ b/src/browsergym/workarena/tasks/mark_duplicate_problem.py @@ -0,0 +1,171 @@ +import json + +from playwright.sync_api import Page +from typing import Tuple + +from .base import AbstractServiceNowTask +from .comp_building_block import CompositionalBuildingBlockTask + +from ..api.utils import table_api_call + + +class SetProblemAsDuplicateTask(AbstractServiceNowTask, CompositionalBuildingBlockTask): + """ + Set a problem as duplicate, assuming we start on the problems list view. + + Parameters: + ----------- + instance: SNowInstance + The instance to use. + start_rel_url: str + The relative URL of the task list. + fixed_config: dict + Configuration to use for the task. If provided, the task will use the provided configuration instead of + selecting a random one. See browsergym/workarena/data_files/task_configs/filter_change_request_list_task.json + for an example of a configuration file. + respect_problem_ordering: bool + Whether to respect the ordering of the problems in the list. If True, the task will pick the first problem in the + list as the target problem. If False, the task validation will check if any problem is a duplicate of the other. + add_comment: bool + Whether or not to add comment to the duplicated task. If set to True, will add "Duplicate" as the problem description + goal_version: str + choice of "base", "priority", "high_priority". Adjusts the goal to the task setting for L2 + """ + + def __init__( + self, + seed: int = None, + instance=None, + start_rel_url="/now/nav/ui/classic/params/target/problem_list.do", + fixed_config: dict = None, + respect_problem_ordering: bool = False, + add_comment: bool = False, + goal_version: str = "base", + level: int = None, + **kwargs, + ) -> None: + super().__init__(seed=seed, instance=instance, start_rel_url=start_rel_url) + self.fixed_config = fixed_config + self.config = fixed_config + + self.problem_sys_id = None + self.respect_problem_ordering = respect_problem_ordering + self.add_comment = add_comment + self.goal_version = goal_version + self.level = level + self.__dict__.update(kwargs) + + def setup_goal(self, page: Page) -> tuple[str, dict]: + self.target_problem = self.fixed_config["target_problem"] + self.source_problem = self.fixed_config["source_problem"] + + goal = self.get_pretty_printed_description() + + return goal, {} + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L2 compositional tasks. + called by subclasses + """ + + if self.level == 3: + task_info = " " + elif self.goal_version == "base": + task_info = "Mark problems with duplicated problem statements as such. You can mark any as duplicate of the other." + elif self.goal_version == "priority": + task_info = "Among the problems with duplicated problem statements, mark the lower priority one as duplicate of the higher priority one" + elif self.goal_version == "high priority": + task_info = "Among the problems with duplicated problem statements, mark any as duplicate of the other. Change the description of the problem marked as duplicate to 'duplicate'." + + return task_info + + def cheat(self, page: Page, chat_messages: list[str]) -> None: + super().cheat(page, chat_messages) + target_problem_number = self.target_problem["number"] + + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + # Search for the private task by search for the number + frame.wait_for_selector(f"[aria-label='Preview record: {target_problem_number}']").click() + page.wait_for_timeout(1500) + # Click on the private task to open it + frame.get_by_text("Open Record").click() + page.wait_for_timeout(2000) + page.wait_for_load_state("networkidle") + frame = page.wait_for_selector('iframe[name="gsft_main"]').content_frame() + page.wait_for_timeout(1500) + # Open the duplicate mode + frame.get_by_text("Mark Duplicate").first.click() + page.wait_for_timeout(1000) + # Close the pop-up to edit the duplicate problem in the same window + frame.get_by_text("Close").last.click() + frame.locator('[aria-labelledby="label.problem.duplicate_of"]').fill( + self.source_problem["number"] + ) + page.keyboard.press("Enter") + page.wait_for_timeout(1000) + if self.add_comment: + frame.locator('[id="problem.description"]').fill("Duplicate") + + frame.get_by_text("update").first.click() + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float, bool, str, dict]: + """ + Validate the solution + """ + target_problem_record = table_api_call( + instance=self.instance, + table="problem", + params={"sysparm_query": f"number={self.target_problem['number']}"}, + )["result"] + source_problem_record = table_api_call( + instance=self.instance, + table="problem", + params={"sysparm_query": f"number={self.source_problem['number']}"}, + )["result"] + # If the ordering can be anything, we check both problems + problem_found = source_problem_record and target_problem_record + + if not problem_found: + return 0, False, "", {"message": "Problem not found in DB."} + + # if the duplicate value is not set, the field will be an empty string; otherwise it will be a dict + target_duplicate_value = target_problem_record[0]["duplicate_of"] + if target_duplicate_value: + target_duplicate_value = target_duplicate_value["value"] + + target_is_duplicate = target_duplicate_value == source_problem_record[0]["sys_id"] + if self.respect_problem_ordering: + problem_marked_as_duplicate = target_is_duplicate + else: + source_duplicate_value = source_problem_record[0]["duplicate_of"] + if source_duplicate_value: + source_duplicate_value = source_duplicate_value["value"] + source_is_duplicate = source_duplicate_value == target_problem_record[0]["sys_id"] + problem_marked_as_duplicate = target_is_duplicate or source_is_duplicate + + if self.add_comment: + comment_added = ( + target_problem_record[0]["description"].lower() == "duplicate" + and target_is_duplicate + ) + if not self.respect_problem_ordering: + comment_added = comment_added or ( + source_problem_record[0]["description"].lower() == "duplicate" + and source_is_duplicate + ) + if not comment_added: + return 0, False, "", {"message": "Comment not added."} + + if not problem_marked_as_duplicate: + return 0, False, "", {"message": "Problem not marked as duplicate."} + + return ( + 1, + True, + "Nice work, thank you!", + {"message": "Problem task was closed as duplicate."}, + ) + + +__TASKS__ = [SetProblemAsDuplicateTask] diff --git a/src/browsergym/workarena/tasks/navigation.py b/src/browsergym/workarena/tasks/navigation.py index e366397..4028da8 100644 --- a/src/browsergym/workarena/tasks/navigation.py +++ b/src/browsergym/workarena/tasks/navigation.py @@ -3,13 +3,14 @@ """ +import json import playwright.sync_api +import re from importlib import resources -import json from playwright.sync_api import Page -from urllib.parse import urlparse, urlunparse, unquote -from typing import Tuple +from urllib import parse +from typing import List, Tuple from ..api.utils import table_api_call from .base import AbstractServiceNowTask @@ -34,11 +35,14 @@ class AllMenuTask(AbstractServiceNowTask): """ - def __init__(self, seed: int, instance: SNowInstance = None, fixed_config: dict = None) -> None: + def __init__( + self, seed: int = None, instance: SNowInstance = None, fixed_config: dict = None, **kwargs + ) -> None: super().__init__(seed=seed, instance=instance, start_rel_url="/now/nav/ui/home") self.fixed_config = fixed_config with open(ALL_MENU_PATH, "r") as f: self.all_configs = json.load(f) + self.__dict__.update(kwargs) def setup_goal(self, page: Page) -> tuple[str, dict]: super().setup_goal(page=page) @@ -47,7 +51,9 @@ def setup_goal(self, page: Page) -> tuple[str, dict]: self.module = ( self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) ) - self.final_url = self.instance.snow_url + self.module["url"] + + # When menu tasks do not need to be validated, the URL can be omitted from their config + self.final_url = self.instance.snow_url + self.module.get("url", "") # Generate goal goal = f'Navigate to the "{self.module["module"]}" module of the "{self.module["application"]}" application.' @@ -55,9 +61,19 @@ def setup_goal(self, page: Page) -> tuple[str, dict]: return goal, info + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + task_info = f'- Navigate to the "{self.module["module"]}" module of the "{self.module["application"]}" application.' + + return task_info + def cheat(self, page: Page, chat_messages: list[str]) -> None: super().cheat(page=page, chat_messages=chat_messages) - + # gsft_main remains undefined on the landing page; we have to wait for the network to be idle instead. + page.wait_for_load_state("networkidle") menu_button = page.locator('div[aria-label="All"]') if menu_button.get_attribute("aria-expanded").lower() != "true": menu_button.click() @@ -100,7 +116,7 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: # In some cases, like System Scheduler > Scheduled Jobs > Scheduled Jobs, modules are repeated in the path # This causes problems when clicking. Therefore, we pick the last item if menu_item.count() > 1: - menu_item = menu_item.last + menu_item = menu_item.first with page.expect_navigation(): menu_item.click() page.wait_for_timeout(2000) @@ -111,11 +127,18 @@ def validate( page.wait_for_load_state("domcontentloaded") # Get the current URL and the final URL - current_url = urlunparse(urlparse(unquote(page.evaluate("() => window.location.href")))) - final_url = urlunparse(urlparse(unquote(self.final_url))) + current_url = parse.urlunparse( + parse.urlparse(parse.unquote(page.evaluate("() => window.location.href"))) + ) + final_url = parse.urlunparse(parse.urlparse(parse.unquote(self.final_url))) if final_url == current_url: - return 1, True, "Nice work, thank you!", {"message": "Correct module reached."} + return ( + 1, + True, + "Nice work, thank you!", + {"message": "Correct module reached."}, + ) return 0, False, "", {"message": "Not at expected URL."} @@ -139,11 +162,14 @@ class ImpersonationTask(AbstractServiceNowTask): """ - def __init__(self, seed: int, instance=None, fixed_config: dict = None) -> None: + def __init__( + self, seed: int = None, instance=None, fixed_config: dict = None, **kwargs + ) -> None: super().__init__(seed=seed, instance=instance, start_rel_url="/now/nav/ui/home") self.fixed_config = fixed_config with open(IMPERSONATION_CONFIG_PATH, "r") as f: self.all_configs = json.load(f) + self.__dict__.update(kwargs) def setup_goal(self, page: Page) -> tuple[str, dict]: super().setup_goal(page=page) @@ -160,6 +186,15 @@ def setup_goal(self, page: Page) -> tuple[str, dict]: return goal, info + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + task_info = f"- Impersonate the user {self.user_full_name} \n" + + return task_info + def cheat(self, page: Page, chat_messages: list[str]) -> None: super().cheat(page=page, chat_messages=chat_messages) impersonate_user(self.user_full_name, page) @@ -167,7 +202,9 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: def validate( self, page: playwright.sync_api.Page, chat_messages: list[str] ) -> Tuple[float, bool, str, dict]: - user_info = self.page.evaluate("window.NOW")["user"] + page.wait_for_function("window.NOW && window.NOW.user") + + user_info = page.evaluate("window.NOW")["user"] # If the current user is not being impersonated, fail. if not user_info["isImpersonating"]: @@ -185,7 +222,12 @@ def validate( # If the name matches, success. if user_fullname == self.user_full_name: - return 1, True, "Nice work, thank you!", {"message": "Correct user impersonated."} + return ( + 1, + True, + "Nice work, thank you!", + {"message": "Correct user impersonated."}, + ) # Otherwise, fail. return 0, False, "", {"message": "Currently impersonating the wrong user."} diff --git a/src/browsergym/workarena/tasks/scripts/extract_all_menu_items.py b/src/browsergym/workarena/tasks/scripts/extract_all_menu_items.py index b9033e5..b0fcf83 100644 --- a/src/browsergym/workarena/tasks/scripts/extract_all_menu_items.py +++ b/src/browsergym/workarena/tasks/scripts/extract_all_menu_items.py @@ -72,7 +72,10 @@ def expand_and_gather_paths(page, parent_selector="body", current_path=[]): expand_and_gather_paths(page, nested_parent_selector, new_path) if not collapsible_lists: - current_path_item = {"path": current_path.copy(), "selector": parent_selector} + current_path_item = { + "path": current_path.copy(), + "selector": parent_selector, + } base_paths.append(current_path_item) def expand_menu(): @@ -190,7 +193,11 @@ def get_application_names(): 45: ] # get only the end of the url if url not in urls: - menu_task = {"application": application, "module": module, "url": url} + menu_task = { + "application": application, + "module": module, + "url": url, + } all_menu_items.append(menu_task) urls[url] = True diff --git a/src/browsergym/workarena/tasks/scripts/generate_dashboard_configs.py b/src/browsergym/workarena/tasks/scripts/generate_dashboard_configs.py index 694d6ea..cd72a77 100644 --- a/src/browsergym/workarena/tasks/scripts/generate_dashboard_configs.py +++ b/src/browsergym/workarena/tasks/scripts/generate_dashboard_configs.py @@ -28,7 +28,7 @@ N_CPU = 20 MAX_CONFIGS = 1000 -REPORT = True # Set to True for reports, False for dashboards +REPORT = False # Set to True for reports, False for dashboards class DummyDashboard(DashboardRetrievalTask): @@ -102,10 +102,10 @@ def get_dashboard_urls(instance): "18b1f472533130104c90ddeeff7b12a6", # Incident overview "287d07d1ff3130106c1ef9a7cddcbd5d", # Request overview "7ab78953eb32011008f2951ff15228e6", # Service catalog overview - "2d297c880f1130101527008c07767e27", # Survey overview + # "2d297c880f1130101527008c07767e27", # Survey overview (almost empty post deleting reports that rely on time) "6b706f448f231110953ddffc9071a4f3", # Telemetry - Table growth - "15c5d2d377213010a435478c4f5a993c", # Usage overview - "85a57f9677100110ba155631dc5a9905", # Web api usage overview + # "15c5d2d377213010a435478c4f5a993c", # Usage overview + # "85a57f9677100110ba155631dc5a9905", # Web api usage overview (empty post deleting reports that rely on time) "c38ca3a273031010ae8dd21efaf6a747", # Data classification "3d48f669538223008329ddeeff7b1253", # Problem overview ] @@ -131,6 +131,7 @@ def get_all_configs_by_url(url, is_report): "chart_series": "", "question": "max", }, + seed=0, ) task.setup(page=page) @@ -196,7 +197,7 @@ def get_all_configs_by_url(url, is_report): ) except Exception as e: print("Exception in worker", url, chart_title, e) - return [] + continue # Skip this chart if len(questions) == 0: return [] diff --git a/src/browsergym/workarena/tasks/scripts/service_catalog.py b/src/browsergym/workarena/tasks/scripts/service_catalog.py index 71b71c3..818f227 100644 --- a/src/browsergym/workarena/tasks/scripts/service_catalog.py +++ b/src/browsergym/workarena/tasks/scripts/service_catalog.py @@ -65,7 +65,8 @@ def generate_configs_for_all_items(): "w", ) as f: all_configs_for_a_single_item = sorted( - all_configs_for_a_single_item, key=lambda x: x["item"] + str(x["quantity"]) + all_configs_for_a_single_item, + key=lambda x: x["item"] + str(x["quantity"]), ) json.dump(all_configs_for_a_single_item, f, indent=4, sort_keys=True) diff --git a/src/browsergym/workarena/tasks/scripts/validate.py b/src/browsergym/workarena/tasks/scripts/validate.py index 12fca72..21805a6 100644 --- a/src/browsergym/workarena/tasks/scripts/validate.py +++ b/src/browsergym/workarena/tasks/scripts/validate.py @@ -156,7 +156,11 @@ def validate_on_page(task_class, task_config, page): def validate_configs( - task_class, config_path, num_tasks: int = None, save_failed_tasks: bool = True, page=None + task_class, + config_path, + num_tasks: int = None, + save_failed_tasks: bool = True, + page=None, ) -> list[dict]: """Validate that the configs are working. Saves failing configs to json so they can be tested.""" with open(config_path, "r") as f: @@ -167,7 +171,9 @@ def validate_configs( failed_tasks = {"cheat": [], "no_reward": [], "exception": [], "not_done": []} with tqdm( - total=len(all_configs), desc=f"Validating {task_class.__name__} configs", ncols=150 + total=len(all_configs), + desc=f"Validating {task_class.__name__} configs", + ncols=150, ) as pbar: for task_config in all_configs: try: diff --git a/src/browsergym/workarena/tasks/send_chat_message.py b/src/browsergym/workarena/tasks/send_chat_message.py new file mode 100644 index 0000000..5504421 --- /dev/null +++ b/src/browsergym/workarena/tasks/send_chat_message.py @@ -0,0 +1,90 @@ +from typing import Tuple +from playwright.sync_api import Page + +from .base import AbstractServiceNowTask +from .comp_building_block import CompositionalBuildingBlockTask + +from ..instance import SNowInstance + + +class SendChatMessageTask(AbstractServiceNowTask, CompositionalBuildingBlockTask): + """Task to send a chat message in the chat. Only used as a compositional building block for the cheat function. + Args: + -------- + message (str): + The message to send in the chat + answer_format (str): + The type of answer to generate. Choice of total_return_only, total_return_and_investments, investments_only, cleanup, cleanup_and_return + """ + + def __init__( + self, + instance: SNowInstance, + message: str, + answer_format: str, + use_description_in_l3: bool = False, + **kwargs, + ): + super().__init__(seed=0, instance=instance, start_rel_url="") + self.message = message + self.answer_format = answer_format + self.use_description_in_l3 = use_description_in_l3 + self.__dict__.update(kwargs) + + def setup_goal(self, page: Page): + return self.get_pretty_printed_description(), {} + + def validate(self, page: Page, chat_messages: list[str]) -> Tuple[float | bool | str | dict]: + return super().validate(page, chat_messages) + + def cheat(self, page: Page, chat_messages: list[str]): + super().cheat(page=page, chat_messages=chat_messages) + chat_messages.append({"role": "assistant", "message": str(self.message)}) + + def teardown(self) -> None: + pass + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in compositional tasks. + """ + raise NotImplementedError + + +class SendChatMessageForBudgetAllocationTask(SendChatMessageTask): + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in compositional tasks. + """ + if self.level == 3: + task_info = "Allocate the budget to maximize revenue." + elif self.level == 2: + task_info = f"Allocate the budget to maximize revenue. This involves going over expense lines and identifying the ones maximizing revenue while fitting in the allowed budget of {self.budget} $. The returns are written in their short description." + if self.answer_format == "total_return_only": + task_info += " Provide only the total return of the investments in the chat." + if self.answer_format == "total_return_and_investments": + task_info += " Provide the total return of the investments as well as the value of their 'Number' field in the chat." + if self.answer_format == "investments_only": + task_info += " Provide only the value of the 'Number' field of the selected investments in the chat." + if self.answer_format == "cleanup": + task_info += " Delete the investments that will not be kept so that only the selected investments remain." + if self.answer_format == "cleanup_and_return": + task_info += " Delete the investments that will not be kept so that only the selected investments remain as well as returning their total value in the chat." + + return task_info + + +class SendChatMessageGenericTask(SendChatMessageTask): + + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in compositional tasks. + """ + if self.use_description_in_l3: + task_info = self.description + elif self.level == 3: + task_info = "" + elif self.level == 2: + task_info = self.description + + return task_info diff --git a/src/browsergym/workarena/tasks/service_catalog.py b/src/browsergym/workarena/tasks/service_catalog.py index bccfc5d..cfece29 100644 --- a/src/browsergym/workarena/tasks/service_catalog.py +++ b/src/browsergym/workarena/tasks/service_catalog.py @@ -10,6 +10,7 @@ import playwright.sync_api from playwright.sync_api import Page +import re from time import sleep from urllib import parse @@ -60,7 +61,10 @@ "iPad mini": { "desc": "Request for iPad mini", "options": { - "Choose the colour": ("radio", ["Space Grey", "Pink", "Purple", "Starlight"]), + "Choose the colour": ( + "radio", + ["Space Grey", "Pink", "Purple", "Starlight"], + ), "Choose the storage": ("radio", ["64", "256"]), }, }, @@ -150,21 +154,25 @@ class OrderHardwareTask(AbstractServiceNowTask): Random seed instance: SNowInstance The instance to use. + fixed_request_item: str + The item to order. If provided, the task will always order this item. fixed_config: dict Configuration to use for the task. If provided, the task will use the provided configuration instead of selecting a random one. See browsergym/workarena/data_files/task_configs/order_ipda_pro_task.json for an example of a configuration file. - config_path: - The path to the JSON file containing all configurations for the task. Provided by subclasses + config_only_in_desc: bool + If True, the model to order will be omitted from the task description in comp tasks. + """ def __init__( self, - seed: int, + seed: int = None, instance: SNowInstance = None, fixed_request_item: str = None, fixed_config: dict = None, - config_path: str = None, + config_only_in_desc: bool = False, + **kwargs, ): super().__init__( seed=seed, @@ -178,12 +186,19 @@ def __init__( raise ValueError(f"'fixed_request_item' and 'fixed_config[\"item\"]' do not match") self.fixed_config = fixed_config + self.config = None self.fixed_request_item = fixed_request_item + self.config_only_in_desc = config_only_in_desc self.js_prefix = "gsft_main" self.js_api_forms = "g_form" - with open(config_path, "r") as f: - self.all_configs = json.load(f) + self.all_configs = self.all_configs() + self.__dict__.update(kwargs) + + @classmethod + def all_configs(cls) -> List[dict]: + with open(cls.config_path, "r") as f: + return json.load(f) def _wait_for_ready(self, page: Page, wait_for_form_api: bool = False) -> None: """ @@ -266,14 +281,19 @@ def setup_goal(self, page: Page) -> tuple[str, dict]: # Get the task configuration assert self.all_configs is not None, "No configuration available for the task." - config = self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) - self.requested_item = config["item"] - self.short_description = config["description"] - self.quantity = config["quantity"] - self.requested_configuration = config["configuration"] + self.config = ( + self.fixed_config if self.fixed_config else self.random.choice(self.all_configs) + ) + self.requested_item = self.config["item"] + self.short_description = self.config["description"] + self.quantity = self.config["quantity"] + self.requested_configuration = self.config["configuration"] # Generate goal - goal = f'Go to the hardware store and order {self.quantity} "{self.requested_item}"' + if self.config_only_in_desc: + goal = self.get_pretty_printed_description() + else: + goal = f'Go to the hardware store and order {self.quantity} "{self.requested_item}"' if len(self.requested_configuration) > 0: goal += f" with configuration {dict((k, v[1]) for k, v in self.requested_configuration.items())}" info = {} @@ -359,6 +379,9 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: def _generate_random_config(self, page: Page): """Generate a random configuration for the task""" + self.task_is_setup = ( + False # This is a hack to avoid raising an exception in the setup method + ) self.setup(page=page, do_start=False) if self.fixed_request_item: self.requested_item = self.fixed_request_item @@ -404,6 +427,39 @@ def _get_control_description(self, page, field): raise ValueError(f"Unknown control type {control_type}") return control_text + def get_pretty_printed_description(self) -> str: + """ + Get the task info for this task when used in a private task; Used in L3 compositional tasks. + called by subclasses + """ + class_name = self.__class__.__name__ + class_name = class_name.replace("Task", "") + # Split the words + words = re.findall(r"[A-Z][^A-Z]*", class_name) + class_name_formatted = " ".join(words) + task_specs = { + "Quantity": self.config["quantity"], + "Configuration": self.config["configuration"], + } + if self.config_only_in_desc: + task_info = f"- Order the item in the following quantities and with the following configuration:\n" + else: + task_specs["Description"] = self.config["description"] + task_info = f"- {class_name_formatted} with the following specifications:\n" + for k, v in task_specs.items(): + # Some values might be empty - like the configuration of the apple watch. It is more natural to exclude them + if not v: + continue + # If the value is a dictionary, print it in a nested way + if isinstance(v, dict): + task_info += f" - {k}:\n" + for k2, v2 in v.items(): + task_info += f" - {k2}: {v2[1]}\n" + else: + task_info += f" - {k}: {v}\n" + + return task_info + def teardown(self) -> None: """ Deletes the request (and automatically all its items) @@ -428,9 +484,7 @@ def validate(self, page: Page, chat_messages: list[str]) -> tuple[int, bool, str ) # Retrieve the request sysid from the URL - current_url = parse.urlparse( - parse.unquote(self.page.evaluate("() => window.location.href")) - ) + current_url = parse.urlparse(parse.unquote(page.evaluate("() => window.location.href"))) (self.request_sysid,) = parse.parse_qs(current_url.query).get("sysparm_sys_id", [None]) if self.request_sysid is None: return ( @@ -496,7 +550,12 @@ def validate(self, page: Page, chat_messages: list[str]) -> tuple[int, bool, str {"message": error_msg}, ) - return 1, True, "Nice work, thank you!", {"message": "Task completed successfully."} + return ( + 1, + True, + "Nice work, thank you!", + {"message": "Task completed successfully."}, + ) def option_match_heuristic(value, option): @@ -510,91 +569,100 @@ def _process(x): class OrderDeveloperLaptopTask(OrderHardwareTask): + config_path = ORDER_DEVELOPER_LAPTOP_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="Developer Laptop (Mac)", - config_path=ORDER_DEVELOPER_LAPTOP_TASK_CONFIG_PATH, **kwargs, ) class OrderIpadMiniTask(OrderHardwareTask): + config_path = ORDER_IPAD_MINI_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="iPad mini", - config_path=ORDER_IPAD_MINI_TASK_CONFIG_PATH, **kwargs, ) class OrderIpadProTask(OrderHardwareTask): + config_path = ORDER_IPAD_PRO_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="iPad pro", - config_path=ORDER_IPAD_PRO_TASK_CONFIG_PATH, **kwargs, ) class OrderSalesLaptopTask(OrderHardwareTask): + config_path = ORDER_SALES_LAPTOP_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="Sales Laptop", - config_path=ORDER_SALES_LAPTOP_TASK_CONFIG_PATH, **kwargs, ) class OrderStandardLaptopTask(OrderHardwareTask): + config_path = ORDER_STANDARD_LAPTOP_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="Standard Laptop", - config_path=ORDER_STANDARD_LAPTOP_TASK_CONFIG_PATH, **kwargs, ) class OrderAppleWatchTask(OrderHardwareTask): + config_path = ORDER_APPLE_WATCH_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="Apple Watch", - config_path=ORDER_APPLE_WATCH_TASK_CONFIG_PATH, **kwargs, ) class OrderAppleMacBookPro15Task(OrderHardwareTask): + config_path = ORDER_APPLE_MAC_BOOK_PRO15_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="Apple MacBook Pro 15", - config_path=ORDER_APPLE_MAC_BOOK_PRO15_TASK_CONFIG_PATH, **kwargs, ) class OrderDevelopmentLaptopPCTask(OrderHardwareTask): + config_path = ORDER_DEVELOPMENT_LAPTOP_PC_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="Development Laptop (PC)", - config_path=ORDER_DEVELOPMENT_LAPTOP_PC_TASK_CONFIG_PATH, **kwargs, ) class OrderLoanerLaptopTask(OrderHardwareTask): + config_path = ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH + def __init__(self, *args, **kwargs): super().__init__( *args, fixed_request_item="Loaner Laptop", - config_path=ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH, **kwargs, ) diff --git a/src/browsergym/workarena/tasks/utils/form.py b/src/browsergym/workarena/tasks/utils/form.py index 3ecb17a..14011f6 100644 --- a/src/browsergym/workarena/tasks/utils/form.py +++ b/src/browsergym/workarena/tasks/utils/form.py @@ -18,9 +18,6 @@ def fill_text(page, input_field, value, iframe=None): The locator of the iframe that contains the input field, by default None """ - if value == "": - return - if iframe is None: iframe = page @@ -28,7 +25,7 @@ def fill_text(page, input_field, value, iframe=None): input_field.click(force=True) # If the field uses autocomplete, we need to wait for Ajax to finish (and expand the menu) - if input_field.get_attribute("aria-autocomplete") == "list": + if input_field.get_attribute("aria-autocomplete") == "list" and value != "": # Fill in the value using a procedure that triggers the autocomplete input_field.fill(value[:-1]) page.keyboard.press(value[-1]) diff --git a/src/browsergym/workarena/tasks/utils/private_tasks.py b/src/browsergym/workarena/tasks/utils/private_tasks.py new file mode 100644 index 0000000..3753b4f --- /dev/null +++ b/src/browsergym/workarena/tasks/utils/private_tasks.py @@ -0,0 +1,63 @@ +import json +import time +from ...api.utils import table_api_call +from playwright.sync_api import Page + + +def create_private_task_and_get_sys_id( + instance, + page: Page, + private_task_id: str, + task_info: str, + short_description: str, + user_sys_id: str = None, +) -> None: + """ + Create a private task in the ServiceNow instance to store the task information. Used for level 3 tasks. + Sets the sys_id of the private task to the sys_id attribute of the task. + Returns the sys_id of the private task. + Parameters: + ---------- + instance: SNowInstance + The instance to use. + page: Page + playwright page + private_task_id: str + ID of the private task to be created. + task_info: str + The information needed to complete the task, written in the private task description. + short_description: str + A short description of the task, written in the private task short description. + user_sys_id: str + The sys_id of the user to assign the task to. If None, the task will be assigned to the admin user. + """ + page.wait_for_load_state("networkidle") + if user_sys_id is None: + # Get the user sys_id; if the page is blank, use the admin user + if page.url == "about:blank": + user_sys_id = table_api_call( + instance=instance, + table="sys_user", + params={"sysparm_query": "user_name=admin"}, + )["result"][0]["sys_id"] + else: + user_sys_id = page.evaluate("() => NOW.user.userID") + # Create private task containing the information needed to complete the task + result = table_api_call( + instance=instance, + table="vtb_task", + data=json.dumps( + { + "number": f"{private_task_id}", + "description": f"{task_info}", + "short_description": f"{short_description}", + "assigned_to": f"{user_sys_id}", + } + ), + method="POST", + )["result"] + sys_id = result["sys_id"] + + assert result, "Failed to create private task" + + return sys_id diff --git a/src/browsergym/workarena/tasks/utils/utils.py b/src/browsergym/workarena/tasks/utils/utils.py index ec1d706..f76616d 100644 --- a/src/browsergym/workarena/tasks/utils/utils.py +++ b/src/browsergym/workarena/tasks/utils/utils.py @@ -18,3 +18,16 @@ def check_url_suffix_match(page: playwright.sync_api.Page, expected_url: str, ta logging.debug(f"Not in the expected URL for {task.__class__.__name__}, but in {page.url}") return False return True + + +def prettyprint_enum(items, conjunction="and"): + """ + Pretty print a list of items with a conjunction + + """ + if not items: + return "" + elif len(items) == 1: + return items[0] + else: + return ", ".join(items[:-1]) + f", {conjunction} " + items[-1] diff --git a/tests/test_api.py b/tests/test_api.py index 889a64d..19e0d1f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,3 +1,4 @@ +import numpy as np import pytest import random diff --git a/tests/test_compositional.py b/tests/test_compositional.py new file mode 100644 index 0000000..145ae07 --- /dev/null +++ b/tests/test_compositional.py @@ -0,0 +1,175 @@ +""" +Tests that are not specific to any particular kind of task. + +""" + +import logging + +import pytest + +# bugfix: use same playwright instance in browsergym and pytest +from utils import setup_playwright + +from playwright.sync_api import Page, TimeoutError +from tenacity import retry, stop_after_attempt, retry_if_exception_type +from browsergym.workarena import ( + ALL_COMPOSITIONAL_TASKS, + get_all_tasks_agents, + get_all_tasks_humans, +) + +AGENT_L2_SAMPLED_SET = get_all_tasks_agents(filter="l2") + +AGENT_L2_SAMPLED_TASKS, AGENT_L2_SEEDS = [sampled_set[0] for sampled_set in AGENT_L2_SAMPLED_SET], [ + sampled_set[1] for sampled_set in AGENT_L2_SAMPLED_SET +] + +AGENT_L3_SAMPLED_SET = get_all_tasks_agents(filter="l3") + +AGENT_L3_SAMPLED_TASKS, AGENT_L3_SEEDS = [sampled_set[0] for sampled_set in AGENT_L3_SAMPLED_SET], [ + sampled_set[1] for sampled_set in AGENT_L3_SAMPLED_SET +] + +HUMAN_L2_SAMPLED_SET = get_all_tasks_humans(filter="l2") + +HUMAN_L2_SAMPLED_TASKS, HUMAN_L2_SEEDS = [sampled_set[0] for sampled_set in HUMAN_L2_SAMPLED_SET], [ + sampled_set[1] for sampled_set in HUMAN_L2_SAMPLED_SET +] + +HUMAN_L3_SAMPLED_SET = get_all_tasks_humans(filter="l3") + +HUMAN_L3_SAMPLED_TASKS, HUMAN_L3_SEEDS = [sampled_set[0] for sampled_set in HUMAN_L3_SAMPLED_SET], [ + sampled_set[1] for sampled_set in HUMAN_L3_SAMPLED_SET +] + + +@retry( + stop=stop_after_attempt(5), + retry=retry_if_exception_type(TimeoutError), + reraise=True, + before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), +) +@pytest.mark.parametrize("task_entrypoint", ALL_COMPOSITIONAL_TASKS) +@pytest.mark.parametrize("random_seed", range(1)) +@pytest.mark.parametrize("level", range(2, 4)) +@pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") +def test_cheat_compositional(task_entrypoint, random_seed, level, page: Page): + task = task_entrypoint(seed=random_seed, level=level) + goal, info = task.setup(page=page) + chat_messages = [] + for i in range(len(task)): + page.wait_for_timeout(1000) + task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) + page.wait_for_timeout(1000) + reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) + if i < len(task) - 1: + assert done is False and reward == 0.0 + + task.teardown() + + assert done is True and reward == 1.0 + + +@retry( + stop=stop_after_attempt(5), + retry=retry_if_exception_type(TimeoutError), + reraise=True, + before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), +) +@pytest.mark.parametrize("task_entrypoint, seed", zip(AGENT_L2_SAMPLED_TASKS, AGENT_L2_SEEDS)) +@pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") +def test_cheat_compositional_sampled_agent_set_l2(task_entrypoint, seed, page: Page): + task = task_entrypoint(seed=seed) + goal, info = task.setup(page=page) + chat_messages = [] + for i in range(len(task)): + page.wait_for_timeout(1000) + task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) + page.wait_for_timeout(1000) + reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) + if i < len(task) - 1: + assert done is False and reward == 0.0 + + task.teardown() + + assert done is True and reward == 1.0 + + +@retry( + stop=stop_after_attempt(5), + retry=retry_if_exception_type(TimeoutError), + reraise=True, + before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), +) +@pytest.mark.parametrize("task_entrypoint, seed", zip(AGENT_L3_SAMPLED_TASKS, AGENT_L3_SEEDS)) +@pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") +def test_cheat_compositional_sampled_agent_set_l3(task_entrypoint, seed, page: Page): + task = task_entrypoint(seed=seed) + goal, info = task.setup(page=page) + chat_messages = [] + for i in range(len(task)): + page.wait_for_timeout(1000) + task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) + page.wait_for_timeout(1000) + reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) + if i < len(task) - 1: + assert done is False and reward == 0.0 + + task.teardown() + + assert done is True and reward == 1.0 + + +@retry( + stop=stop_after_attempt(5), + retry=retry_if_exception_type(TimeoutError), + reraise=True, + before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), +) +@pytest.mark.parametrize("task_entrypoint, seed", zip(HUMAN_L2_SAMPLED_TASKS, HUMAN_L2_SEEDS)) +@pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") +def test_cheat_compositional_sampled_human_set_l2(task_entrypoint, seed, page: Page): + task = task_entrypoint(seed=seed) + goal, info = task.setup(page=page) + chat_messages = [] + for i in range(len(task)): + page.wait_for_timeout(1000) + task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) + page.wait_for_timeout(1000) + reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) + if i < len(task) - 1: + assert done is False and reward == 0.0 + + task.teardown() + + assert done is True and reward == 1.0 + + +@retry( + stop=stop_after_attempt(5), + retry=retry_if_exception_type(TimeoutError), + reraise=True, + before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), +) +@pytest.mark.parametrize("task_entrypoint, seed", zip(HUMAN_L3_SAMPLED_TASKS, HUMAN_L3_SEEDS)) +@pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") +def test_cheat_compositional_sampled_human_set_l3(task_entrypoint, seed, page: Page): + task = task_entrypoint(seed=seed) + goal, info = task.setup(page=page) + chat_messages = [] + for i in range(len(task)): + page.wait_for_timeout(1000) + task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) + page.wait_for_timeout(1000) + reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) + if i < len(task) - 1: + assert done is False and reward == 0.0 + + task.teardown() + + assert done is True and reward == 1.0 diff --git a/tests/test_compositional_utils.py b/tests/test_compositional_utils.py new file mode 100644 index 0000000..3406d32 --- /dev/null +++ b/tests/test_compositional_utils.py @@ -0,0 +1,92 @@ +from copy import deepcopy +import json +import numpy as np +import pytest + +from browsergym.workarena.tasks.compositional.utils.knapsack import KnapsackInstanceGenarator +from browsergym.workarena.tasks.compositional.utils.infeasible_configs import ( + get_infeasible_form_config, + get_infeasible_service_catalog_config, + get_infeasible_filter_config, + get_infeasible_sort_config, +) +from browsergym.workarena.config import ( + CREATE_USER_CONFIG_PATH, + ORDER_APPLE_MAC_BOOK_PRO15_TASK_CONFIG_PATH, + FILTER_USER_LIST_CONFIG_PATH, + SORT_USER_LIST_CONFIG_PATH, +) + + +@pytest.mark.parametrize( + "mode", ["random", "trivial", "single_item", "single_item_uniform", "n_items"] +) +def test_knapsack(mode: str, num_items_in_solution: int = 2): + num_items_in_solution = 2 if mode == "n_items" else 1 + knapsack = KnapsackInstanceGenarator( + random=np.random, + num_items=3, + max_capacity=150000, + mode=mode, + num_items_in_solution=num_items_in_solution, + ) + investments, max_return, selected_indices = knapsack.get_instance() + + # In these modes, all items are identical, so the optimal solution can be any + if mode in ["n_items", "single_item_uniform"]: + selected_indices = [i for i in range(num_items_in_solution)] + + assert len(investments) == 3 + assert sum(investments[i][0] for i in selected_indices) <= 150000 + assert max_return == sum(investments[i][1] for i in selected_indices) + + if mode != "trivial": + unselected_index = [i for i in range(3) if i not in selected_indices][0] + assert ( + sum(investments[i][0] for i in selected_indices) + investments[unselected_index][0] + > 150000 + ) + else: + assert len(selected_indices) == len(investments) + + +config_generator_and_config_path = [ + [get_infeasible_form_config, CREATE_USER_CONFIG_PATH], + [get_infeasible_service_catalog_config, ORDER_APPLE_MAC_BOOK_PRO15_TASK_CONFIG_PATH], + [get_infeasible_filter_config, FILTER_USER_LIST_CONFIG_PATH], + [get_infeasible_sort_config, SORT_USER_LIST_CONFIG_PATH], +] + + +@pytest.mark.parametrize("function_to_path", config_generator_and_config_path) +def test_invalid_config_generator(function_to_path): + def parse_nested_dict(nested_dict, keywords): + """Look for keywords in a nested dictionary. + Return True if any keyword is found, False otherwise. + """ + for key, value in nested_dict.items(): + if key in keywords or value in keywords: + return True + if isinstance(value, dict): + if parse_nested_dict(value, keywords): + return True + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + if parse_nested_dict(item, keywords): + return True + elif isinstance(value, str): + for keyword in keywords: + if keyword in value: + return True + return False + + config_generator, config_path = function_to_path + with open(config_path, "r") as f: + config = json.load(f)[0] + base_config = deepcopy(config) + + invalid_config, infeasible_reasons = config_generator(random=np.random, config=config) + assert invalid_config != base_config + assert parse_nested_dict(invalid_config, infeasible_reasons) + assert parse_nested_dict(base_config, infeasible_reasons) == False diff --git a/tests/test_random_config_generation.py b/tests/test_random_config_generation.py index 8de36c3..f3c4ab0 100644 --- a/tests/test_random_config_generation.py +++ b/tests/test_random_config_generation.py @@ -48,23 +48,23 @@ ) RANDOMLY_CONFIGURALBE_TASKS = [ - # CreateChangeRequestTask, - # CreateHardwareAssetTask, - # CreateIncidentTask, - # CreateProblemTask, - # CreateUserTask, - # FilterAssetListTask, - # FilterChangeRequestListTask, - # FilterHardwareListTask, - # FilterIncidentListTask, - # FilterServiceCatalogItemListTask, - # FilterUserListTask, - # SortAssetListTask, - # SortChangeRequestListTask, - # SortHardwareListTask, - # SortIncidentListTask, - # SortServiceCatalogItemListTask, - # SortUserListTask, + CreateChangeRequestTask, + CreateHardwareAssetTask, + CreateIncidentTask, + CreateProblemTask, + CreateUserTask, + FilterAssetListTask, + FilterChangeRequestListTask, + FilterHardwareListTask, + FilterIncidentListTask, + FilterServiceCatalogItemListTask, + FilterUserListTask, + SortAssetListTask, + SortChangeRequestListTask, + SortHardwareListTask, + SortIncidentListTask, + SortServiceCatalogItemListTask, + SortUserListTask, OrderDeveloperLaptopTask, OrderIpadMiniTask, OrderIpadProTask, @@ -77,15 +77,16 @@ ] -# @retry( -# stop=stop_after_attempt(5), -# retry=retry_if_exception_type(TimeoutError), -# reraise=True, -# before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), -# ) +@retry( + stop=stop_after_attempt(5), + retry=retry_if_exception_type(TimeoutError), + reraise=True, + before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), +) @pytest.mark.parametrize("task_entrypoint", RANDOMLY_CONFIGURALBE_TASKS) @pytest.mark.parametrize("random_seed", range(1)) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_cheat_from_random_config(task_entrypoint, random_seed: int, page: Page): task = task_entrypoint(seed=random_seed) task._generate_random_config(page=page) diff --git a/tests/test_task_from_config.py b/tests/test_task_from_config.py index 27a9bf4..50b4656 100644 --- a/tests/test_task_from_config.py +++ b/tests/test_task_from_config.py @@ -113,12 +113,14 @@ def generic_task_cheat_test(task_class, config_path, page: Page, expected_goal: # Navigation tasks @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_menu_task_from_config(page: Page): expected_goal = 'Navigate to the "AI Search for Next Experience > Guided Setup for Zing to AI Search Migration" module of the "AI Search" application.' generic_task_cheat_test(AllMenuTask, ALL_MENU_PATH, page, expected_goal=expected_goal) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_impersonation_from_config(page: Page): expected_goal = "Impersonate the user ATF Change Management." generic_task_cheat_test( @@ -128,39 +130,49 @@ def test_impersonation_from_config(page: Page): # Service Catalog tasks @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_developer_laptop_task_from_config(page: Page): generic_task_cheat_test(OrderDeveloperLaptopTask, ORDER_DEVELOPER_LAPTOP_TASK_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_ipad_mini_task_from_config(page: Page): generic_task_cheat_test(OrderIpadMiniTask, ORDER_IPAD_MINI_TASK_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_ipad_pro_task_from_config(page: Page): generic_task_cheat_test(OrderIpadProTask, ORDER_IPAD_PRO_TASK_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_sales_laptop_task_from_config(page: Page): generic_task_cheat_test(OrderSalesLaptopTask, ORDER_SALES_LAPTOP_TASK_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_standard_laptop_task_from_config(page: Page): generic_task_cheat_test(OrderStandardLaptopTask, ORDER_STANDARD_LAPTOP_TASK_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_apple_watch_task_from_config(page: Page): expected_goal = 'Go to the hardware store and order 1 "Apple Watch"' generic_task_cheat_test( - OrderAppleWatchTask, ORDER_APPLE_WATCH_TASK_CONFIG_PATH, page, expected_goal=expected_goal + OrderAppleWatchTask, + ORDER_APPLE_WATCH_TASK_CONFIG_PATH, + page, + expected_goal=expected_goal, ) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_apple_macbook_pro15_task_from_config(page: Page): generic_task_cheat_test( OrderAppleMacBookPro15Task, ORDER_APPLE_MAC_BOOK_PRO15_TASK_CONFIG_PATH, page @@ -168,6 +180,7 @@ def test_order_apple_macbook_pro15_task_from_config(page: Page): @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_development_laptop_pc_task_from_config(page: Page): generic_task_cheat_test( OrderDevelopmentLaptopPCTask, ORDER_DEVELOPMENT_LAPTOP_PC_TASK_CONFIG_PATH, page @@ -175,41 +188,48 @@ def test_order_development_laptop_pc_task_from_config(page: Page): @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_order_loaner_laptop_task_from_config(page: Page): generic_task_cheat_test(OrderLoanerLaptopTask, ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH, page) # form tasks @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_create_change_request_task_from_config(page: Page): generic_task_cheat_test(CreateChangeRequestTask, CREATE_CHANGE_REQUEST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_create_hardware_asset_task_from_config(page: Page): generic_task_cheat_test(CreateHardwareAssetTask, CREATE_HARDWARE_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_create_incident_task_from_config(page: Page): generic_task_cheat_test(CreateIncidentTask, CREATE_INCIDENT_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_create_problem_task_from_config(page: Page): - expected_goal = 'Create a new problem with a value of "Request for a Blackberry" for field "Problem statement" and a value of "3 - Low" for field "Impact" and a value of "" for field "Service" and a value of "Hardware" for field "Category" and a value of "bizonal wateringly nonsuccessful checkerberry abridgeable" for field "Description" and a value of "" for field "Configuration item".' + expected_goal = 'Create a new problem with a value of "Request for a Blackberry" for field "Problem statement", a value of "3 - Low" for field "Impact", a value of "" for field "Service", a value of "Hardware" for field "Category", a value of "bizonal wateringly nonsuccessful checkerberry abridgeable" for field "Description", and a value of "" for field "Configuration item".' generic_task_cheat_test( CreateProblemTask, CREATE_PROBLEM_CONFIG_PATH, page, expected_goal=expected_goal ) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_create_user_task_from_config(page: Page): generic_task_cheat_test(CreateUserTask, CREATE_USER_CONFIG_PATH, page) # knowledge tasks @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_knowledge_base_from_config(page: Page): expected_goal = "Answer the following question using the knowledge base: \"Can you provide the direct contact number for the CEO? Answer with the full phone number starting with the '+' sign.\"" generic_task_cheat_test(KnowledgeBaseSearchTask, KB_CONFIG_PATH, page) @@ -217,11 +237,13 @@ def test_knowledge_base_from_config(page: Page): # list tasks @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_filter_asset_list_task_from_config(page: Page): generic_task_cheat_test(FilterAssetListTask, FILTER_ASSET_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_filter_change_request_list_task_from_config(page: Page): expected_goal = 'Create a filter for the list to extract all entries where "Assignment group" is "Hardware" and "Assigned to" is "Bow Ruggeri".' generic_task_cheat_test( @@ -233,48 +255,59 @@ def test_filter_change_request_list_task_from_config(page: Page): @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_filter_hardware_list_task_from_config(page: Page): generic_task_cheat_test(FilterHardwareListTask, FILTER_HARDWARE_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_filter_incident_list_task_from_config(page: Page): generic_task_cheat_test(FilterIncidentListTask, FILTER_INCIDENT_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_filter_service_catalog_item_list_task_from_config(page: Page): generic_task_cheat_test( - FilterServiceCatalogItemListTask, FILTER_SERVICE_CATALOG_ITEM_LIST_CONFIG_PATH, page + FilterServiceCatalogItemListTask, + FILTER_SERVICE_CATALOG_ITEM_LIST_CONFIG_PATH, + page, ) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_filter_user_list_task_from_config(page: Page): generic_task_cheat_test(FilterUserListTask, FILTER_USER_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_sort_asset_list_task_from_config(page: Page): generic_task_cheat_test(SortAssetListTask, SORT_ASSET_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_sort_change_request_list_task_from_config(page: Page): generic_task_cheat_test(SortChangeRequestListTask, SORT_CHANGE_REQUEST_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_sort_hardware_list_task_from_config(page: Page): generic_task_cheat_test(SortHardwareListTask, SORT_HARDWARE_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_sort_incident_list_task_from_config(page: Page): generic_task_cheat_test(SortIncidentListTask, SORT_INCIDENT_LIST_CONFIG_PATH, page) @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_sort_service_catalog_item_list_task_from_config(page: Page): generic_task_cheat_test( SortServiceCatalogItemListTask, SORT_SERVICE_CATALOG_ITEM_LIST_CONFIG_PATH, page @@ -282,6 +315,7 @@ def test_sort_service_catalog_item_list_task_from_config(page: Page): @pytest.mark.slow +@pytest.mark.skip(reason="Tests are too slow") def test_sort_user_list_task_from_config(page: Page): expected_goal = 'Sort the "users" list by the following fields:\n - Active (descending)' generic_task_cheat_test( diff --git a/tests/test_task_general.py b/tests/test_task_general.py index 3358b32..b410003 100644 --- a/tests/test_task_general.py +++ b/tests/test_task_general.py @@ -14,7 +14,7 @@ from playwright.sync_api import Page, TimeoutError from tenacity import retry, stop_after_attempt, retry_if_exception_type -from browsergym.workarena import ALL_WORKARENA_TASKS +from browsergym.workarena import ATOMIC_TASKS @retry( @@ -23,7 +23,7 @@ reraise=True, before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), ) -@pytest.mark.parametrize("task_entrypoint", ALL_WORKARENA_TASKS) +@pytest.mark.parametrize("task_entrypoint", ATOMIC_TASKS) @pytest.mark.parametrize("random_seed", range(1)) @pytest.mark.slow def test_cheat(task_entrypoint, random_seed: int, page: Page): @@ -32,13 +32,8 @@ def test_cheat(task_entrypoint, random_seed: int, page: Page): chat_messages = [] reward, done, message, info = task.validate(page, chat_messages) assert done is False and reward == 0.0 - assert ( - isinstance(reward, (int, float)) - and type(done) == bool - and type(message) == str - and type(info) == dict - ) + assert type(message) == str and type(info) == dict task.cheat(page=page, chat_messages=chat_messages) reward, done, message, info = task.validate(page, chat_messages) - assert done is True and reward == 1.0 task.teardown() + assert done is True and reward == 1.0 From 912e4305187d4773bf586e908759b959267577ff Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Fri, 14 Jun 2024 16:31:09 -0400 Subject: [PATCH 04/11] version bump 0.3.0 --- src/browsergym/workarena/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/browsergym/workarena/__init__.py b/src/browsergym/workarena/__init__.py index 1444d17..28bf906 100644 --- a/src/browsergym/workarena/__init__.py +++ b/src/browsergym/workarena/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.0dev" +__version__ = "0.3.0" import inspect import numpy as np From 58c85da0e0567f36742ddf1c4c0921fc34a755f8 Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Fri, 14 Jun 2024 16:35:01 -0400 Subject: [PATCH 05/11] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45a34de..3088cc8 100644 --- a/README.md +++ b/README.md @@ -152,4 +152,4 @@ Please use the following BibTeX to cite our work: archivePrefix={arXiv}, primaryClass={cs.LG} } -``` +``` \ No newline at end of file From c38c228c0841f09147e4b19b20427fcee7b2f1d4 Mon Sep 17 00:00:00 2001 From: Leo Boisvert Date: Mon, 17 Jun 2024 08:38:29 -0400 Subject: [PATCH 06/11] remove compositional tasks from bGym registration --- src/browsergym/workarena/__init__.py | 101 +--------------- tests/test_compositional.py | 175 --------------------------- 2 files changed, 1 insertion(+), 275 deletions(-) delete mode 100644 tests/test_compositional.py diff --git a/src/browsergym/workarena/__init__.py b/src/browsergym/workarena/__init__.py index 28bf906..574a731 100644 --- a/src/browsergym/workarena/__init__.py +++ b/src/browsergym/workarena/__init__.py @@ -11,30 +11,16 @@ from .tasks.knowledge import __TASKS__ as KB_TASKS from .tasks.list import __TASKS__ as LIST_TASKS from .tasks.navigation import __TASKS__ as NAVIGATION_TASKS -from .tasks.compositional.base import CompositionalTask -from .tasks.compositional.update_task import __TASKS__ as UPDATE_TASKS -from .tasks.compositional import ( - ALL_COMPOSITIONAL_TASKS, - ALL_COMPOSITIONAL_TASKS_L2, - ALL_COMPOSITIONAL_TASKS_L3, - AGENT_CURRICULUM_L2, - AGENT_CURRICULUM_L3, - HUMAN_CURRICULUM_L2, - HUMAN_CURRICULUM_L3, -) -from .tasks.compositional.base import HumanEvalTask from .tasks.service_catalog import __TASKS__ as SERVICE_CATALOG_TASKS +from .tasks.compositional.base import CompositionalTask ALL_WORKARENA_TASKS = [ - *ALL_COMPOSITIONAL_TASKS_L2, - *ALL_COMPOSITIONAL_TASKS_L3, *DASHBOARD_TASKS, *FORM_TASKS, *KB_TASKS, *LIST_TASKS, *NAVIGATION_TASKS, *SERVICE_CATALOG_TASKS, - *UPDATE_TASKS, ] ATOMIC_TASKS = [ task @@ -44,94 +30,9 @@ and not issubclass(task, CompositionalBuildingBlockTask) ] - # register the WorkArena benchmark for task in ALL_WORKARENA_TASKS: register_task( task.get_task_id(), task, ) - - -def get_all_tasks_agents(filter="l2", meta_seed=42, n_seed_l1=10): - all_task_tuples = [] - filter = filter.split(".") - if len(filter) > 2: - raise Exception("Unsupported filter used.") - if len(filter) == 1: - level = filter[0] - if level not in ["l1", "l2", "l3"]: - raise Exception("Unsupported category of tasks.") - else: - rng = np.random.RandomState(meta_seed) - if level == "l1": - for task in ATOMIC_TASKS: - for seed in rng.randint(0, 1000, n_seed_l1): - all_task_tuples.append((task, int(seed))) - - return all_task_tuples - - if len(filter) == 2: - level, filter_category = filter[0], filter[1] - if filter_category not in list(AGENT_CURRICULUM_L2.keys()): - raise Exception("Unsupported category of tasks.") - else: - filter_category = None - - if level == "l2": - ALL_COMPOSITIONAL_TASKS_CATEGORIES = AGENT_CURRICULUM_L2 - else: - ALL_COMPOSITIONAL_TASKS_CATEGORIES = AGENT_CURRICULUM_L3 - - for category, items in ALL_COMPOSITIONAL_TASKS_CATEGORIES.items(): - if filter_category and category != filter_category: - continue - for curr_seed in rng.randint(0, 1000, items["num_seeds"]): - random_gen = np.random.RandomState(curr_seed) - for task_set, count in zip(items["buckets"], items["weights"]): - tasks = random_gen.choice(task_set, count, replace=False) - for task in tasks: - all_task_tuples.append((task, curr_seed)) - - return all_task_tuples - - -def get_all_tasks_humans(filter="l2", meta_seed=42): - OFFSET = 42 - all_task_tuples = [] - filter = filter.split(".") - if len(filter) > 2: - raise Exception("Unsupported filter used.") - if len(filter) == 1: - level = filter[0] - if level not in ["l1", "l2", "l3"]: - raise Exception("Unsupported category of tasks.") - else: - rng = np.random.RandomState(meta_seed) - if level == "l1": - return [(task, rng.randint(0, 1000)) for task in ATOMIC_TASKS] - - if len(filter) == 2: - level, filter_category = filter[0], filter[1] - if filter_category not in list(HUMAN_CURRICULUM_L2.keys()): - raise Exception("Unsupported category of tasks.") - else: - filter_category = None - - if level == "l2": - ALL_COMPOSITIONAL_TASKS_CATEGORIES = HUMAN_CURRICULUM_L2 - else: - ALL_COMPOSITIONAL_TASKS_CATEGORIES = HUMAN_CURRICULUM_L3 - - for category, items in ALL_COMPOSITIONAL_TASKS_CATEGORIES.items(): - if filter_category and category != filter_category: - continue - # We will come back to this after the submission - for curr_seed in rng.randint(0, 1000, items["num_seeds"]): - random_gen = np.random.RandomState(curr_seed) - for task_set, count in zip(items["buckets"], items["weights"]): - tasks = random_gen.choice(task_set, count, replace=False) - for task in tasks: - all_task_tuples.append((task, curr_seed)) - - return all_task_tuples diff --git a/tests/test_compositional.py b/tests/test_compositional.py deleted file mode 100644 index 145ae07..0000000 --- a/tests/test_compositional.py +++ /dev/null @@ -1,175 +0,0 @@ -""" -Tests that are not specific to any particular kind of task. - -""" - -import logging - -import pytest - -# bugfix: use same playwright instance in browsergym and pytest -from utils import setup_playwright - -from playwright.sync_api import Page, TimeoutError -from tenacity import retry, stop_after_attempt, retry_if_exception_type -from browsergym.workarena import ( - ALL_COMPOSITIONAL_TASKS, - get_all_tasks_agents, - get_all_tasks_humans, -) - -AGENT_L2_SAMPLED_SET = get_all_tasks_agents(filter="l2") - -AGENT_L2_SAMPLED_TASKS, AGENT_L2_SEEDS = [sampled_set[0] for sampled_set in AGENT_L2_SAMPLED_SET], [ - sampled_set[1] for sampled_set in AGENT_L2_SAMPLED_SET -] - -AGENT_L3_SAMPLED_SET = get_all_tasks_agents(filter="l3") - -AGENT_L3_SAMPLED_TASKS, AGENT_L3_SEEDS = [sampled_set[0] for sampled_set in AGENT_L3_SAMPLED_SET], [ - sampled_set[1] for sampled_set in AGENT_L3_SAMPLED_SET -] - -HUMAN_L2_SAMPLED_SET = get_all_tasks_humans(filter="l2") - -HUMAN_L2_SAMPLED_TASKS, HUMAN_L2_SEEDS = [sampled_set[0] for sampled_set in HUMAN_L2_SAMPLED_SET], [ - sampled_set[1] for sampled_set in HUMAN_L2_SAMPLED_SET -] - -HUMAN_L3_SAMPLED_SET = get_all_tasks_humans(filter="l3") - -HUMAN_L3_SAMPLED_TASKS, HUMAN_L3_SEEDS = [sampled_set[0] for sampled_set in HUMAN_L3_SAMPLED_SET], [ - sampled_set[1] for sampled_set in HUMAN_L3_SAMPLED_SET -] - - -@retry( - stop=stop_after_attempt(5), - retry=retry_if_exception_type(TimeoutError), - reraise=True, - before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), -) -@pytest.mark.parametrize("task_entrypoint", ALL_COMPOSITIONAL_TASKS) -@pytest.mark.parametrize("random_seed", range(1)) -@pytest.mark.parametrize("level", range(2, 4)) -@pytest.mark.slow -@pytest.mark.skip(reason="Tests are too slow") -def test_cheat_compositional(task_entrypoint, random_seed, level, page: Page): - task = task_entrypoint(seed=random_seed, level=level) - goal, info = task.setup(page=page) - chat_messages = [] - for i in range(len(task)): - page.wait_for_timeout(1000) - task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) - page.wait_for_timeout(1000) - reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) - if i < len(task) - 1: - assert done is False and reward == 0.0 - - task.teardown() - - assert done is True and reward == 1.0 - - -@retry( - stop=stop_after_attempt(5), - retry=retry_if_exception_type(TimeoutError), - reraise=True, - before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), -) -@pytest.mark.parametrize("task_entrypoint, seed", zip(AGENT_L2_SAMPLED_TASKS, AGENT_L2_SEEDS)) -@pytest.mark.slow -@pytest.mark.skip(reason="Tests are too slow") -def test_cheat_compositional_sampled_agent_set_l2(task_entrypoint, seed, page: Page): - task = task_entrypoint(seed=seed) - goal, info = task.setup(page=page) - chat_messages = [] - for i in range(len(task)): - page.wait_for_timeout(1000) - task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) - page.wait_for_timeout(1000) - reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) - if i < len(task) - 1: - assert done is False and reward == 0.0 - - task.teardown() - - assert done is True and reward == 1.0 - - -@retry( - stop=stop_after_attempt(5), - retry=retry_if_exception_type(TimeoutError), - reraise=True, - before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), -) -@pytest.mark.parametrize("task_entrypoint, seed", zip(AGENT_L3_SAMPLED_TASKS, AGENT_L3_SEEDS)) -@pytest.mark.slow -@pytest.mark.skip(reason="Tests are too slow") -def test_cheat_compositional_sampled_agent_set_l3(task_entrypoint, seed, page: Page): - task = task_entrypoint(seed=seed) - goal, info = task.setup(page=page) - chat_messages = [] - for i in range(len(task)): - page.wait_for_timeout(1000) - task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) - page.wait_for_timeout(1000) - reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) - if i < len(task) - 1: - assert done is False and reward == 0.0 - - task.teardown() - - assert done is True and reward == 1.0 - - -@retry( - stop=stop_after_attempt(5), - retry=retry_if_exception_type(TimeoutError), - reraise=True, - before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), -) -@pytest.mark.parametrize("task_entrypoint, seed", zip(HUMAN_L2_SAMPLED_TASKS, HUMAN_L2_SEEDS)) -@pytest.mark.slow -@pytest.mark.skip(reason="Tests are too slow") -def test_cheat_compositional_sampled_human_set_l2(task_entrypoint, seed, page: Page): - task = task_entrypoint(seed=seed) - goal, info = task.setup(page=page) - chat_messages = [] - for i in range(len(task)): - page.wait_for_timeout(1000) - task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) - page.wait_for_timeout(1000) - reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) - if i < len(task) - 1: - assert done is False and reward == 0.0 - - task.teardown() - - assert done is True and reward == 1.0 - - -@retry( - stop=stop_after_attempt(5), - retry=retry_if_exception_type(TimeoutError), - reraise=True, - before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."), -) -@pytest.mark.parametrize("task_entrypoint, seed", zip(HUMAN_L3_SAMPLED_TASKS, HUMAN_L3_SEEDS)) -@pytest.mark.slow -@pytest.mark.skip(reason="Tests are too slow") -def test_cheat_compositional_sampled_human_set_l3(task_entrypoint, seed, page: Page): - task = task_entrypoint(seed=seed) - goal, info = task.setup(page=page) - chat_messages = [] - for i in range(len(task)): - page.wait_for_timeout(1000) - task.cheat(page=page, chat_messages=chat_messages, subtask_idx=i) - page.wait_for_timeout(1000) - reward, done, message, info = task.validate(page=page, chat_messages=chat_messages) - if i < len(task) - 1: - assert done is False and reward == 0.0 - - task.teardown() - - assert done is True and reward == 1.0 From 2807a0a852207f29bfdae41721822b1c5c3db611 Mon Sep 17 00:00:00 2001 From: Alexandre Drouin Date: Mon, 17 Jun 2024 09:20:45 -0400 Subject: [PATCH 07/11] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3088cc8..a15b237 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ https://github.com/ServiceNow/WorkArena/assets/1726818/ca26dfaf-2358-4418-855f-8 **Goal:** The agent must answer a question that requires reading charts and (optionally) performing simple reasoning over them. +*Note: For demonstration purposes, a human is controlling the cursor since this is a pure retrieval task* + https://github.com/ServiceNow/WorkArena/assets/1726818/0023232c-081f-4be4-99bd-f60c766e6c3f @@ -152,4 +154,4 @@ Please use the following BibTeX to cite our work: archivePrefix={arXiv}, primaryClass={cs.LG} } -``` \ No newline at end of file +``` From df926c00e6cf86579b922b398b3b57930aa55089 Mon Sep 17 00:00:00 2001 From: Alexandre Drouin Date: Mon, 17 Jun 2024 11:13:51 -0400 Subject: [PATCH 08/11] Fix issues in demo code --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a15b237..7bebbc7 100644 --- a/README.md +++ b/README.md @@ -114,14 +114,13 @@ for task in ALL_WORKARENA_TASKS: # Instantiate a new environment env = BrowserEnv(task_entrypoint=task, - headless=False, - slow_mo=1000) + headless=False) env.reset() # Cheat functions use Playwright to automatically solve the task env.chat.add_message(role="assistant", msg="On it. Please wait...") cheat_messages = [] - env.task.cheat(env.page, messages) + env.task.cheat(env.page, cheat_messages) # Send cheat messages to chat for cheat_msg in cheat_messages: @@ -131,7 +130,7 @@ for task in ALL_WORKARENA_TASKS: env.chat.add_message(role="assistant", msg="I'm done!") # Validate the solution - reward, stop, message, info = env.task.validate(env.page, env.chat.messages) + reward, stop, message, info = env.task.validate(env.page, cheat_messages) if reward == 1: env.chat.add_message(role="user", msg="Yes, that works. Thanks!") else: From 729745bbcfed3bc5f8a2d78f84b9e1e28a8c4a1b Mon Sep 17 00:00:00 2001 From: Alexandre Drouin Date: Mon, 17 Jun 2024 11:21:08 -0400 Subject: [PATCH 09/11] Add clarification docstring --- scripts/make_human_eval_curriculum.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/make_human_eval_curriculum.py b/scripts/make_human_eval_curriculum.py index ae4a7eb..e8d6d2a 100644 --- a/scripts/make_human_eval_curriculum.py +++ b/scripts/make_human_eval_curriculum.py @@ -1,3 +1,12 @@ +""" +Human Evaluation - Create the curriculum for all humans + +Note: This script separates the tasks among 14 evaluators. + A 15th one was added subsequently to solve tasks that + had not been completed by the initial 14 (e.g., due + to some issues with the annotation UI). + +""" import random from browsergym.workarena import get_all_tasks_humans @@ -14,7 +23,7 @@ "parikh", "marchand", "paquet", - "nayal", + "nayak", "huang", "subbaraj", "williams", From 5b709ad0dca704d56cfd9826e70c997f4f9f2806 Mon Sep 17 00:00:00 2001 From: Alexandre Drouin Date: Mon, 17 Jun 2024 11:22:07 -0400 Subject: [PATCH 10/11] Rename to be more explicit --- scripts/{wa_action_traces.py => extract_finetuning_traces.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename scripts/{wa_action_traces.py => extract_finetuning_traces.py} (98%) diff --git a/scripts/wa_action_traces.py b/scripts/extract_finetuning_traces.py similarity index 98% rename from scripts/wa_action_traces.py rename to scripts/extract_finetuning_traces.py index 8ff5b84..4d4e676 100644 --- a/scripts/wa_action_traces.py +++ b/scripts/extract_finetuning_traces.py @@ -1,5 +1,5 @@ """ -A demonstration of how action traces (with observations) can be extracted +A demonstration of how observation/action traces can be extracted for WorkArena tasks without modifying the task code. Author: Alexandre Drouin (alexandre.drouin@servicenow.com) From 5408afae5e005ff247db42cb5943022fb818d3fa Mon Sep 17 00:00:00 2001 From: Maxime Gasse Date: Mon, 17 Jun 2024 11:22:56 -0400 Subject: [PATCH 11/11] black format --- scripts/make_human_eval_curriculum.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/make_human_eval_curriculum.py b/scripts/make_human_eval_curriculum.py index e8d6d2a..4e8370e 100644 --- a/scripts/make_human_eval_curriculum.py +++ b/scripts/make_human_eval_curriculum.py @@ -7,6 +7,7 @@ to some issues with the annotation UI). """ + import random from browsergym.workarena import get_all_tasks_humans