Skip to content

Commit

Permalink
added check for new data updates
Browse files Browse the repository at this point in the history
  • Loading branch information
Azaya89 committed May 27, 2024
1 parent 939e56e commit eae9999
Showing 1 changed file with 20 additions and 3 deletions.
23 changes: 20 additions & 3 deletions update_data.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import gspread
import logging
import os
import pandas as pd
import requests
import subprocess
Expand All @@ -8,6 +9,16 @@
# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')


def read_existing_data(file_path):
"""Reads existing data from a CSV file to compare with newly fetched data."""
if os.path.exists(file_path):
logging.info("Existing data file found. Reading data...")
return pd.read_csv(file_path)
else:
logging.info("No existing data file found. Assuming first run.")
return pd.DataFrame()

def download_sheet(sheet_id, range_name):
"""Downloads data from Google Sheets and returns a DataFrame."""
try:
Expand All @@ -24,7 +35,7 @@ def download_sheet(sheet_id, range_name):
sheet = client.open_by_key(sheet_id)

# Get the sheet by name
worksheet = sheet.get_worksheet(0)
worksheet = sheet.get_worksheet(range_name)

# Get all records of the data
data = worksheet.get_all_records()
Expand Down Expand Up @@ -72,8 +83,14 @@ def git_commit_push():
RANGE_NAME = 'data'
FILE_PATH = 'omoku_data.csv'

df = download_sheet(SHEET_ID, RANGE_NAME)
save_to_csv(df, FILE_PATH)
new_data = download_sheet(SHEET_ID, RANGE_NAME)
existing_data = read_existing_data(FILE_PATH)

if not new_data.empty and new_data.equals(existing_data):
logging.info("No new data to update.")
exit(0)

save_to_csv(new_data, FILE_PATH)
git_commit_push()
except Exception as e:
logging.critical(f"An unexpected error occurred: {e}")
Expand Down

0 comments on commit eae9999

Please sign in to comment.