From 8cd443893254a1d7742ef49df0c8ed713ec16772 Mon Sep 17 00:00:00 2001 From: rclarke0 Date: Tue, 9 Apr 2024 10:14:24 -0400 Subject: [PATCH] jira ticket adds url to abr-run-data sheet --- .../automation/google_sheets_tool.py | 8 ++- .../abr_testing/automation/jira_tool.py | 11 +-- .../data_collection/abr_calibration_logs.py | 30 ++++----- .../data_collection/abr_google_drive.py | 26 ++++--- .../abr_testing/data_collection/abr_lpc.py | 1 + .../data_collection/abr_robot_error.py | 67 ++++++++++++++++--- .../data_collection/read_robot_logs.py | 35 ++++++---- abr-testing/abr_testing/tools/abr_scale.py | 1 + 8 files changed, 126 insertions(+), 53 deletions(-) diff --git a/abr-testing/abr_testing/automation/google_sheets_tool.py b/abr-testing/abr_testing/automation/google_sheets_tool.py index e5c746e914d..af38a39dcc0 100644 --- a/abr-testing/abr_testing/automation/google_sheets_tool.py +++ b/abr-testing/abr_testing/automation/google_sheets_tool.py @@ -56,10 +56,14 @@ def write_header(self, header: List) -> None: def write_to_row(self, data: List) -> None: """Write data into a row in a List[] format.""" - try: self.row_index += 1 - data = [item.strftime("%Y/%m/%d %H:%M:%S") if isinstance(item, datetime) else item for item in data] + data = [ + item.strftime("%Y/%m/%d %H:%M:%S") + if isinstance(item, datetime) + else item + for item in data + ] self.worksheet.insert_row(data, index=self.row_index) except socket.gaierror: pass diff --git a/abr-testing/abr_testing/automation/jira_tool.py b/abr-testing/abr_testing/automation/jira_tool.py index aff3a6798c3..5c0a2556dfb 100644 --- a/abr-testing/abr_testing/automation/jira_tool.py +++ b/abr-testing/abr_testing/automation/jira_tool.py @@ -5,7 +5,7 @@ import json import webbrowser import argparse -from typing import List, Tuple +from typing import List class JiraTicket: @@ -41,11 +41,12 @@ def issues_on_board(self, board_id: str) -> List[str]: issue_ids.append(issue_id) return issue_ids - def open_issue(self, issue_key: str) -> None: + def open_issue(self, issue_key: str) -> str: """Open issue on web browser.""" url = f"{self.url}/browse/{issue_key}" print(f"Opening at {url}.") webbrowser.open(url) + return url def create_ticket( self, @@ -58,7 +59,7 @@ def create_ticket( components: list, affects_versions: str, robot: str, - ) -> Tuple[str, str]: + ) -> str: """Create ticket.""" data = { "fields": { @@ -94,13 +95,15 @@ def create_ticket( response_str = str(response.content) issue_url = response.json().get("self") issue_key = response.json().get("key") + print(f"issue key {issue_key}") + print(f"issue url{issue_url}") if issue_key is None: print("Error: Could not create issue. No key returned.") except requests.exceptions.HTTPError: print(f"HTTP error occurred. Response content: {response_str}") except json.JSONDecodeError: print(f"JSON decoding error occurred. Response content: {response_str}") - return issue_url, issue_key + return issue_key def post_attachment_to_ticket(self, issue_id: str, attachment_path: str) -> None: """Adds attachments to ticket.""" diff --git a/abr-testing/abr_testing/data_collection/abr_calibration_logs.py b/abr-testing/abr_testing/data_collection/abr_calibration_logs.py index 0923fd7076b..4d744b5b2f5 100644 --- a/abr-testing/abr_testing/data_collection/abr_calibration_logs.py +++ b/abr-testing/abr_testing/data_collection/abr_calibration_logs.py @@ -1,5 +1,5 @@ """Get Calibration logs from robots.""" -from typing import Dict, Any, List +from typing import Dict, Any, List, Union import argparse import os import json @@ -16,7 +16,7 @@ def check_for_duplicates( col_2: int, row: List[str], headers: List[str], -) -> List[str]: +) -> Union[List[str], None]: """Check google sheet for duplicates.""" serials = google_sheet.get_column(col_1) modify_dates = google_sheet.get_column(col_2) @@ -67,7 +67,7 @@ def upload_calibration_offsets( instrument_row, instrument_headers, ) - + # MODULE SHEET if len(calibration.get("Modules", "")) > 0: module_headers = ( @@ -202,19 +202,19 @@ def upload_calibration_offsets( except FileNotFoundError: print(f"Add .json file with robot IPs to: {storage_directory}.") sys.exit() - + if ip_or_all == "ALL": - ip_address_list = ip_file["ip_address_list"] - for ip in ip_address_list: - print(ip) - try: - saved_file_path, calibration = read_robot_logs.get_calibration_offsets( - ip, storage_directory - ) - upload_calibration_offsets(calibration, storage_directory) - except Exception: - print(f"ERROR: Failed to read IP address: {ip}") - continue + ip_address_list = ip_file["ip_address_list"] + for ip in ip_address_list: + print(ip) + try: + saved_file_path, calibration = read_robot_logs.get_calibration_offsets( + ip, storage_directory + ) + upload_calibration_offsets(calibration, storage_directory) + except Exception: + print(f"ERROR: Failed to read IP address: {ip}") + continue else: saved_file_path, calibration = read_robot_logs.get_calibration_offsets( ip_or_all, storage_directory diff --git a/abr-testing/abr_testing/data_collection/abr_google_drive.py b/abr-testing/abr_testing/data_collection/abr_google_drive.py index f39c4426f67..6470f1e0410 100644 --- a/abr-testing/abr_testing/data_collection/abr_google_drive.py +++ b/abr-testing/abr_testing/data_collection/abr_google_drive.py @@ -6,7 +6,7 @@ import gspread # type: ignore[import] from datetime import datetime, timedelta from abr_testing.data_collection import read_robot_logs -from typing import Set, Dict, Any, Tuple, List +from typing import Set, Dict, Any, Tuple, List, Union from abr_testing.automation import google_drive_tool, google_sheets_tool @@ -30,7 +30,9 @@ def get_modules(file_results: Dict[str, str]) -> Dict[str, Any]: def create_data_dictionary( - runs_to_save: Set[str], storage_directory: str, + runs_to_save: Union[Set[str], str], + storage_directory: str, + issue_url: str, ) -> Tuple[Dict[Any, Dict[str, Any]], List]: """Pull data from run files and format into a dictionary.""" runs_and_robots = {} @@ -41,7 +43,7 @@ def create_data_dictionary( file_results = json.load(file) else: continue - run_id = file_results.get("run_id") + run_id = file_results.get("run_id", "NaN") if run_id in runs_to_save: robot = file_results.get("robot_name") protocol_name = file_results["protocol"]["metadata"].get("protocolName", "") @@ -56,7 +58,7 @@ def create_data_dictionary( error_instrument, error_level, ) = read_robot_logs.get_error_info(file_results) - + all_modules = get_modules(file_results) start_time_str, complete_time_str, start_date, run_time_min = ( @@ -104,13 +106,14 @@ def create_data_dictionary( tc_dict = read_robot_logs.thermocycler_commands(file_results) hs_dict = read_robot_logs.hs_commands(file_results) tm_dict = read_robot_logs.temperature_module_commands(file_results) - notes = {"Note1": "", "Jira Link": ""} + notes = {"Note1": "", "Jira Link": issue_url} row_2 = {**row, **all_modules, **notes, **hs_dict, **tm_dict, **tc_dict} headers = list(row_2.keys()) runs_and_robots[run_id] = row_2 else: - os.remove(file_path) - print(f"Run ID: {run_id} has a run time of 0 minutes. Run removed.") + continue + # os.remove(file_path) + # print(f"Run ID: {run_id} has a run time of 0 minutes. Run removed.") return runs_and_robots, headers @@ -169,8 +172,10 @@ def create_data_dictionary( except gspread.exceptions.APIError: print("ERROR: Check google sheet name. Check credentials file.") sys.exit() - try: - google_sheet_lpc = google_sheets_tool.google_sheet(credentials_path, "ABR-LPC", 0) + try: + google_sheet_lpc = google_sheets_tool.google_sheet( + credentials_path, "ABR-LPC", 0 + ) print("Connected to google sheet ABR-LPC") except gspread.exceptions.APIError: print("ERROR: Check google sheet name. Check credentials file.") @@ -188,9 +193,8 @@ def create_data_dictionary( ) # Add missing runs to google sheet runs_and_robots, headers = create_data_dictionary( - missing_runs_from_gs, storage_directory + missing_runs_from_gs, storage_directory, "" ) read_robot_logs.write_to_local_and_google_sheet( runs_and_robots, storage_directory, google_sheet_name, google_sheet, headers ) - \ No newline at end of file diff --git a/abr-testing/abr_testing/data_collection/abr_lpc.py b/abr-testing/abr_testing/data_collection/abr_lpc.py index e69de29bb2d..39aedb2749e 100644 --- a/abr-testing/abr_testing/data_collection/abr_lpc.py +++ b/abr-testing/abr_testing/data_collection/abr_lpc.py @@ -0,0 +1 @@ +"""Get LPC Values from Run logs and pair with calibration.""" diff --git a/abr-testing/abr_testing/data_collection/abr_robot_error.py b/abr-testing/abr_testing/data_collection/abr_robot_error.py index 292a946bd98..b139b5a3ade 100644 --- a/abr-testing/abr_testing/data_collection/abr_robot_error.py +++ b/abr-testing/abr_testing/data_collection/abr_robot_error.py @@ -3,7 +3,13 @@ from abr_testing.data_collection import read_robot_logs, abr_google_drive, get_run_logs import requests import argparse -from abr_testing.automation import jira_tool +from abr_testing.automation import jira_tool, google_sheets_tool, google_drive_tool +import shutil +import os +import subprocess +import json +import sys +import gspread # type: ignore[import] def get_error_runs_from_robot(ip: str) -> List[str]: @@ -139,19 +145,19 @@ def get_error_info_from_robot( affects_version, components, whole_description_str, - saved_file_path, + run_log_file_path, ) = get_error_info_from_robot(ip, one_run, storage_directory) # get calibration data saved_file_path_calibration, calibration = read_robot_logs.get_calibration_offsets( ip, storage_directory ) - read_robot_logs.get_logs(storage_directory, ip) + file_paths = read_robot_logs.get_logs(storage_directory, ip) print(f"Making ticket for run: {one_run} on robot {robot}.") # TODO: make argument or see if I can get rid of with using board_id. project_key = "RABR" parent_key = project_key + "-" + robot[-1] - issues_ids = ticket.issues_on_board(board_id) - issue_url, issue_key = ticket.create_ticket( + # CREATE TICKET + issue_key = ticket.create_ticket( summary, whole_description_str, project_key, @@ -162,6 +168,51 @@ def get_error_info_from_robot( affects_version, parent_key, ) - ticket.open_issue(issue_key) - ticket.post_attachment_to_ticket(issue_key, saved_file_path) - ticket.post_attachment_to_ticket(issue_key, saved_file_path_calibration) + # OPEN TICKET + issue_url = ticket.open_issue(issue_key) + # MOVE FILES TO ERROR FOLDER. + error_files = [saved_file_path_calibration, run_log_file_path] + file_paths + error_folder_path = os.path.join(storage_directory, str("RABR-238")) + os.makedirs(error_folder_path, exist_ok=True) + for source_file in error_files: + destination_file = os.path.join( + error_folder_path, os.path.basename(source_file) + ) + shutil.move(source_file, destination_file) + # OPEN FOLDER DIRECTORY + subprocess.Popen(["explorer", error_folder_path]) + # CONNECT TO GOOGLE DRIVE + credentials_path = os.path.join(storage_directory, "credentials.json") + google_sheet_name = "ABR-run-data" + try: + google_drive = google_drive_tool.google_drive( + credentials_path, + "1Cvej0eadFOTZr9ILRXJ0Wg65ymOtxL4m", + "rhyann.clarke@opentrons.ocm", + ) + print("Connected to google drive.") + except json.decoder.JSONDecodeError: + print( + "Credential file is damaged. Get from https://console.cloud.google.com/apis/credentials" + ) + sys.exit() + # CONNECT TO GOOGLE SHEET + try: + google_sheet = google_sheets_tool.google_sheet( + credentials_path, google_sheet_name, 0 + ) + print(f"Connected to google sheet: {google_sheet_name}") + except gspread.exceptions.APIError: + print("ERROR: Check google sheet name. Check credentials file.") + sys.exit() + # WRITE ERRORED RUN TO GOOGLE SHEET + error_run_log = os.path.join(error_folder_path, os.path.basename(run_log_file_path)) + google_drive.upload_file(error_run_log) + run_id = os.path.basename(error_run_log).split("_")[1].split(".")[0] + runs_and_robots, headers = abr_google_drive.create_data_dictionary( + run_id, error_folder_path, issue_url + ) + read_robot_logs.write_to_local_and_google_sheet( + runs_and_robots, storage_directory, google_sheet_name, google_sheet, headers + ) + print("Wrote run to ABR-run-data") diff --git a/abr-testing/abr_testing/data_collection/read_robot_logs.py b/abr-testing/abr_testing/data_collection/read_robot_logs.py index dbbeb02386a..6f07c1c28a1 100644 --- a/abr-testing/abr_testing/data_collection/read_robot_logs.py +++ b/abr-testing/abr_testing/data_collection/read_robot_logs.py @@ -13,7 +13,8 @@ import json import requests -def lpc_data(file_results: Dict[str, Any], protocol_info: Dict)-> List[Dict[str, Any]]: + +def lpc_data(file_results: Dict[str, Any], protocol_info: Dict) -> List[Dict[str, Any]]: """Get labware offsets from one run log.""" offsets = file_results.get("labwareOffsets", "") all_offsets: List[Dict[str, Any]] = [] @@ -30,17 +31,17 @@ def lpc_data(file_results: Dict[str, Any], protocol_info: Dict)-> List[Dict[str, row = { "createdAt": created_at, "Labware Type": labware_type, - "Slot": slot, - "Module": module_location, + "Slot": slot, + "Module": module_location, "Adapter": adapter, "X": x_offset, - "Y": y_offset, - "Z": z_offset + "Y": y_offset, + "Z": z_offset, } row2 = {**protocol_info, **row} all_offsets.append(row2) return all_offsets - + def command_time(command: Dict[str, str]) -> Tuple[float, float]: """Calculate total create and complete time per command.""" @@ -110,7 +111,7 @@ def hs_commands(file_results: Dict[str, Any]) -> Dict[str, float]: temp_time = datetime.strptime( command.get("completedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z" ) - hs_latch_sets = hs_latch_count / 2 # one set of open/close + hs_latch_sets = hs_latch_count / 2 # one set of open/close hs_total_rotations = sum(hs_rotations.values()) hs_total_temp_time = sum(hs_temps.values()) hs_dict = { @@ -455,12 +456,20 @@ def get_calibration_offsets( json.dump(calibration, open(saved_file_path, mode="w")) return saved_file_path, calibration -def get_logs(storage_directory: str, ip: str): + +def get_logs(storage_directory: str, ip: str) -> List[str]: """Get Robot logs.""" - log_types = ["api.log", "server.log", "serial.log"] + log_types = ["api.log", "server.log", "serial.log", "touchscreen.log"] + all_paths = [] for log_type in log_types: - response = requests.get(f"http://{ip}:31950/logs", headers={"log_identifier": log_type},params={"cursor": 0, "pageLength": 0}) - log_name = ip + "-" + log_type.split(".")[0] + ".json" + response = requests.get( + f"http://{ip}:31950/logs", + headers={"log_identifier": log_type}, + params={"cursor": 0, "pageLength": 0}, + ) + log_name = ip + "_" + log_type.split(".")[0] + ".json" log_data = response.json() - file_name = os.path.join(storage_directory, log_name) - json.dump(log_data, open(file_name, mode="w")) + file_path = os.path.join(storage_directory, log_name) + json.dump(log_data, open(file_path, mode="w")) + all_paths.append(file_path) + return all_paths diff --git a/abr-testing/abr_testing/tools/abr_scale.py b/abr-testing/abr_testing/tools/abr_scale.py index db0a106df6f..75c887d4ecc 100644 --- a/abr-testing/abr_testing/tools/abr_scale.py +++ b/abr-testing/abr_testing/tools/abr_scale.py @@ -78,6 +78,7 @@ is_stable = False break_all = False while is_stable is False: + grams, is_stable = scale.read_mass() grams, is_stable = scale.read_mass() print(f"Scale reading: grams={grams}, is_stable={is_stable}") time_now = datetime.datetime.now()