added script to read individual files (#15028)

<!--
Thanks for taking the time to open a pull request! Please make sure
you've read the "Opening Pull Requests" section of our Contributing
Guide:


https://github.com/Opentrons/opentrons/blob/edge/CONTRIBUTING.md#opening-pull-requests

To ensure your code is reviewed quickly and thoroughly, please fill out
the sections below to the best of your ability!
-->

# Overview

Local Run Log Reader

# Test Plan

Tested on a folder of multiple run logs. 
-->

# Changelog

Added functions in google drive tool to download and search for files
Added a script to look for files based on a local .json file with
strings and download
Added a script to allow for local run log reading without a google
sheets upload.

# Review requests

<!--
Describe any requests for your reviewers here.
-->

# Risk assessment

<!--
Carefully go over your pull request and look at the other parts of the
codebase it may affect. Look for the possibility, even if you think it's
small, that your change may affect some other part of the system - for
instance, changing return tip behavior in protocol may also change the
behavior of labware calibration.

Identify the other parts of the system your codebase may affect, so that
in addition to your own review and testing, other people who may not
have the system internalized as much as you can focus their attention
and testing there.
-->

abr-testing/abr_testing/automation/google_drive_tool.py

@@ -1,12 +1,16 @@
 """Google Drive Tool."""
 import os
-from typing import Set, Any, Optional
+import io
+import json
+import sys
+from typing import Set, Any, Optional, List, Dict
 import webbrowser
 import mimetypes
 from oauth2client.service_account import ServiceAccountCredentials  # type: ignore[import]
 import googleapiclient  # type: ignore[import]
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaFileUpload
+from googleapiclient.http import MediaIoBaseDownload
 
 """Google Drive Tool.
 
@@ -19,13 +23,17 @@ class google_drive:
 
     def __init__(self, credentials: Any, folder_name: str, email: str) -> None:
         """Connects to google drive via credentials file."""
-        self.scope = ["https://www.googleapis.com/auth/drive"]
-        self.credentials = ServiceAccountCredentials.from_json_keyfile_name(
-            credentials, self.scope
-        )
-        self.drive_service = build("drive", "v3", credentials=self.credentials)
-        self.parent_folder = folder_name
-        self.email = email
+        try:
+            self.scope = ["https://www.googleapis.com/auth/drive"]
+            self.credentials = ServiceAccountCredentials.from_json_keyfile_name(
+                credentials, self.scope
+            )
+            self.drive_service = build("drive", "v3", credentials=self.credentials)
+            self.parent_folder = folder_name
+            self.email = email
+        except json.decoder.JSONDecodeError:
+            print("Error! Get file: https://console.cloud.google.com/apis/credentials")
+            sys.exit()
 
     def list_folder(self, delete: Any = False) -> Set[str]:
         """List folders and files in Google Drive."""
@@ -88,7 +96,7 @@ def upload_file(self, file_path: str) -> str:
 
     def upload_missing_files(self, storage_directory: str) -> None:
         """Upload missing files to Google Drive."""
-        # Read Google Drive .json files.
+        # Read .json files.
         google_drive_files = self.list_folder()
         google_drive_files_json = [
             file for file in google_drive_files if file.endswith(".json")
@@ -111,7 +119,7 @@ def upload_missing_files(self, storage_directory: str) -> None:
             except googleapiclient.errors.HttpError:
                 continue
 
-        # Fetch the updated file list after all files are uploaded
+        # Fetch the updated file list after all are uploaded
         files = google_drive.list_folder(self)
 
         file_names = [file for file in files]
@@ -122,9 +130,7 @@ def upload_missing_files(self, storage_directory: str) -> None:
                     f"File '{this_name}' was successfully uploaded with ID: {uploaded_file['id']}"
                 )
             else:
-                print(
-                    f"File '{this_name}' was not found in the list of files after uploading."
-                )
+                print(f"File '{this_name}' was not found after uploading.")
         print(f"{len(files)} item(s) in Google Drive")
 
     def open_folder(self) -> Optional[str]:
@@ -154,3 +160,32 @@ def share_permissions(self, file_id: str) -> None:
         self.drive_service.permissions().create(
             fileId=file_id, body=new_permission, transferOwnership=False  # type: ignore
         ).execute()
+
+    def download_files(
+        self, files_to_download: List[Dict[str, Any]], save_directory: str
+    ) -> None:
+        """Download files to a specified directory."""
+        for file in files_to_download:
+            id = file["id"]
+            file_name = file["name"]
+            file_path = os.path.join(save_directory, file_name)
+            request = self.drive_service.files().get_media(fileId=id)  # type: ignore[attr-defined]
+            fh = io.FileIO(file_path, "wb")
+            downloader = MediaIoBaseDownload(fh, request)
+            done = False
+            while done is False:
+                status, done = downloader.next_chunk()
+                print(f"Downloading {file_name}... {int(status.progress() * 100)}%")
+
+    def search_folder(self, search_strings: List[str], folder_id: str) -> List[Any]:
+        """Search folder for files containing string from list."""
+        files_found = []
+        for search_string in search_strings:
+            query = f"'{folder_id}' in parents and name contains '{search_string}'"
+            response = (
+                self.drive_service.files()
+                .list(q=query, fields="files(id,name)")
+                .execute()
+            )
+            files_found.extend(response.get("files", []))
+        return files_found

abr-testing/abr_testing/automation/google_sheets_tool.py

@@ -3,6 +3,7 @@
 import socket
 import httplib2
 import time as t
+import sys
 from datetime import datetime
 from oauth2client.service_account import ServiceAccountCredentials  # type: ignore[import]
 from typing import Dict, List, Any, Set, Tuple
@@ -19,19 +20,24 @@ class google_sheet:
 
     def __init__(self, credentials: Any, file_name: str, tab_number: int) -> None:
         """Connects to google sheet via credentials file."""
-        self.scope = [
-            "https://spreadsheets.google.com/feeds",
-            "https://www.googleapis.com/auth/drive",
-        ]
-        self.credentials = ServiceAccountCredentials.from_json_keyfile_name(
-            credentials, self.scope
-        )
-        self.gc = gspread.authorize(self.credentials)
-        self.file_name = file_name
-        self.tab_number = tab_number
-        self.spread_sheet = self.open_google_sheet()
-        self.worksheet = self.open_worksheet(self.tab_number)
-        self.row_index = 1
+        try:
+            self.scope = [
+                "https://spreadsheets.google.com/feeds",
+                "https://www.googleapis.com/auth/drive",
+            ]
+            self.credentials = ServiceAccountCredentials.from_json_keyfile_name(
+                credentials, self.scope
+            )
+            self.gc = gspread.authorize(self.credentials)
+            self.file_name = file_name
+            self.tab_number = tab_number
+            self.spread_sheet = self.open_google_sheet()
+            self.worksheet = self.open_worksheet(self.tab_number)
+            self.row_index = 1
+            print(f"Connected to google sheet: {self.file_name}")
+        except gspread.exceptions.APIError:
+            print("ERROR: Check google sheet name. Check credentials file.")
+            sys.exit()
 
     def open_google_sheet(self) -> Any:
         """Open Google Spread Sheet."""
@@ -79,7 +85,7 @@ def write_to_row(self, data: List) -> None:
 
     def delete_row(self, row_index: int) -> None:
         """Delete Row from google sheet."""
-        self.worksheet.delete_row(row_index)
+        self.worksheet.delete_rows(row_index)
 
     def update_cell(
         self, row: int, column: int, single_data: Any
@@ -125,3 +131,13 @@ def token_check(self) -> None:
         """Check if still credentials are still logged in."""
         if self.credentials.access_token_expired:
             self.gc.login()
+
+    def get_row_index_with_value(self, some_string: str, col_num: int) -> Any:
+        """Find row index of string by looking in specific column."""
+        cell = self.worksheet.find(some_string, in_column=col_num)
+        try:
+            row_index = int(cell.row)
+        except AttributeError:
+            print("Row not found.")
+            return None
+        return row_index

abr-testing/abr_testing/data_collection/abr_calibration_logs.py

@@ -3,7 +3,6 @@
 import argparse
 import os
 import json
-import gspread  # type: ignore[import]
 import sys
 import time as t
 from abr_testing.data_collection import read_robot_logs
@@ -175,32 +174,17 @@ def upload_calibration_offsets(
     except FileNotFoundError:
         print(f"Add credentials.json file to: {storage_directory}.")
         sys.exit()
-    try:
-        google_drive = google_drive_tool.google_drive(
-            credentials_path, folder_name, email
-        )
-        # Upload calibration logs to google drive.
-        print("Connected to google drive.")
-    except json.decoder.JSONDecodeError:
-        print(
-            "Credential file is damaged. Get from https://console.cloud.google.com/apis/credentials"
-        )
-        sys.exit()
+    google_drive = google_drive_tool.google_drive(credentials_path, folder_name, email)
     # Connect to google sheet
-    try:
-        google_sheet_instruments = google_sheets_tool.google_sheet(
-            credentials_path, google_sheet_name, 0
-        )
-        google_sheet_modules = google_sheets_tool.google_sheet(
-            credentials_path, google_sheet_name, 1
-        )
-        google_sheet_deck = google_sheets_tool.google_sheet(
-            credentials_path, google_sheet_name, 2
-        )
-        print(f"Connected to google sheet: {google_sheet_name}")
-    except gspread.exceptions.APIError:
-        print("ERROR: Check google sheet name. Check credentials file.")
-        sys.exit()
+    google_sheet_instruments = google_sheets_tool.google_sheet(
+        credentials_path, google_sheet_name, 0
+    )
+    google_sheet_modules = google_sheets_tool.google_sheet(
+        credentials_path, google_sheet_name, 1
+    )
+    google_sheet_deck = google_sheets_tool.google_sheet(
+        credentials_path, google_sheet_name, 2
+    )
     ip_json_file = os.path.join(storage_directory, "IPs.json")
     try:
         ip_file = json.load(open(ip_json_file))

abr-testing/abr_testing/data_collection/abr_google_drive.py

@@ -3,7 +3,6 @@
 import os
 import sys
 import json
-import gspread  # type: ignore[import]
 from datetime import datetime, timedelta
 from abr_testing.data_collection import read_robot_logs
 from typing import Set, Dict, Any, Tuple, List, Union
@@ -121,8 +120,6 @@ def create_data_dictionary(
                 runs_and_robots[run_id] = row_2
             else:
                 continue
-                # os.remove(file_path)
-                # print(f"Run ID: {run_id} has a run time of 0 minutes. Run removed.")
     return runs_and_robots, headers
 
 
@@ -162,33 +159,13 @@ def create_data_dictionary(
     except FileNotFoundError:
         print(f"Add credentials.json file to: {storage_directory}.")
         sys.exit()
-    try:
-        google_drive = google_drive_tool.google_drive(
-            credentials_path, folder_name, email
-        )
-        print("Connected to google drive.")
-    except json.decoder.JSONDecodeError:
-        print(
-            "Credential file is damaged. Get from https://console.cloud.google.com/apis/credentials"
-        )
-        sys.exit()
+    google_drive = google_drive_tool.google_drive(credentials_path, folder_name, email)
     # Get run ids on google sheet
-    try:
-        google_sheet = google_sheets_tool.google_sheet(
-            credentials_path, google_sheet_name, 0
-        )
-        print(f"Connected to google sheet: {google_sheet_name}")
-    except gspread.exceptions.APIError:
-        print("ERROR: Check google sheet name. Check credentials file.")
-        sys.exit()
-    try:
-        google_sheet_lpc = google_sheets_tool.google_sheet(
-            credentials_path, "ABR-LPC", 0
-        )
-        print("Connected to google sheet ABR-LPC")
-    except gspread.exceptions.APIError:
-        print("ERROR: Check google sheet name. Check credentials file.")
-        sys.exit()
+    google_sheet = google_sheets_tool.google_sheet(
+        credentials_path, google_sheet_name, 0
+    )
+    google_sheet_lpc = google_sheets_tool.google_sheet(credentials_path, "ABR-LPC", 0)
+
     run_ids_on_gs = google_sheet.get_column(2)
     run_ids_on_gs = set(run_ids_on_gs)
 

abr-testing/abr_testing/data_collection/get_run_logs.py

@@ -139,14 +139,5 @@ def get_all_run_logs(storage_directory: str) -> None:
     except FileNotFoundError:
         print(f"Add credentials.json file to: {storage_directory}.")
         sys.exit()
-    try:
-        google_drive = google_drive_tool.google_drive(
-            credentials_path, folder_name, email
-        )
-        print("Connected to google drive.")
-    except json.decoder.JSONDecodeError:
-        print(
-            "Credential file is damaged. Get from https://console.cloud.google.com/apis/credentials"
-        )
-        sys.exit()
+    google_drive = google_drive_tool.google_drive(credentials_path, folder_name, email)
     get_all_run_logs(storage_directory)

abr-testing/abr_testing/data_collection/module_ramp_rates.py

@@ -1,7 +1,6 @@
 """Get ramp rates of modules."""
 from abr_testing.automation import google_sheets_tool
 from abr_testing.data_collection import read_robot_logs
-import gspread  # type: ignore[import]
 import argparse
 import os
 import sys
@@ -79,14 +78,9 @@ def ramp_rate(file_results: Dict[str, Any]) -> Dict[int, float]:
         print(f"Add credentials.json file to: {storage_directory}.")
         sys.exit()
     # CONNECT TO GOOGLE SHEET
-    try:
-        google_sheet = google_sheets_tool.google_sheet(
-            credentials_path, google_sheet_name, 1
-        )
-        print(f"Connected to google sheet: {google_sheet_name}")
-    except gspread.exceptions.APIError:
-        print("ERROR: Check google sheet name. Check credentials file.")
-        sys.exit()
+    google_sheet = google_sheets_tool.google_sheet(
+        credentials_path, google_sheet_name, 1
+    )
     run_ids_on_sheet = google_sheet.get_column(2)
     runs_and_robots = {}
     for filename in os.listdir(storage_directory):

abr-testing/abr_testing/data_collection/read_robot_logs.py

@@ -299,11 +299,10 @@ def get_error_info(file_results: Dict[str, Any]) -> Tuple[int, str, str, str, st
             # Instrument Error
             error_instrument = run_command_error["error"]["errorInfo"]["node"]
         except KeyError:
-            # Module Error
+            # Module
             error_instrument = run_command_error["error"]["errorInfo"].get("port", "")
     else:
         error_type = file_results["errors"][0]["errorType"]
-        print(error_type)
         error_code = file_results["errors"][0]["errorCode"]
         error_instrument = file_results["errors"][0]["detail"]
     for error in error_levels:

abr-testing/abr_testing/data_collection/single_run_log_reader.py

@@ -0,0 +1,51 @@
+"""Reads single run log retrieved by get_run_logs.py and saves to local csv."""
+import argparse
+import sys
+import os
+import csv
+from abr_testing.data_collection import read_robot_logs
+from abr_testing.data_collection import abr_google_drive
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Read single run log locally saved.")
+    parser.add_argument(
+        "run_log_file_path",
+        metavar="RUN_LOG_FILE_PATH",
+        type=str,
+        nargs=1,
+        help="Folder path that holds individual run logs of interest.",
+    )
+    parser.add_argument(
+        "google_sheet_name",
+        metavar="GOOGLE_SHEET_NAME",
+        type=str,
+        nargs=1,
+        help="Google sheet name.",
+    )
+    args = parser.parse_args()
+    run_log_file_path = args.run_log_file_path[0]
+    google_sheet_name = args.google_sheet_name[0]
+
+    try:
+        credentials_path = os.path.join(run_log_file_path, "credentials.json")
+    except FileNotFoundError:
+        print(f"Add credentials.json file to: {run_log_file_path}.")
+        sys.exit()
+    # Get Runs from Storage and Read Logs
+    run_ids_in_storage = read_robot_logs.get_run_ids_from_storage(run_log_file_path)
+    runs_and_robots, header = abr_google_drive.create_data_dictionary(
+        run_ids_in_storage, run_log_file_path, ""
+    )
+    list_of_runs = list(runs_and_robots.keys())
+    # Adds Run to local csv
+    sheet_location = os.path.join(run_log_file_path, "saved_data.csv")
+    file_exists = os.path.exists(sheet_location) and os.path.getsize(sheet_location) > 0
+    with open(sheet_location, "a", newline="") as f:
+        writer = csv.writer(f)
+        if not file_exists:
+            writer.writerow(header)
+        for run in list_of_runs:
+            # Add new row
+            row = runs_and_robots[run].values()
+            row_list = list(row)
+            writer.writerow(row_list)