From 3a2d1ab61d85d2d2b594f5be907cfd6c66cfcd95 Mon Sep 17 00:00:00 2001 From: Zander Otavka Date: Tue, 5 Jan 2016 14:41:16 -0800 Subject: [PATCH] Loop through the API and improve search/sort. Now, a request is almost guaranteed to fill to maxResults, if there is also a nextPageToken. --- api/__init__.py | 2 ++ api/authutils.py | 2 ++ api/calendarsapi.py | 2 ++ api/eventsapi.py | 44 +++++++++++++++-------- api/gapiutils.py | 2 ++ api/garbagecollect.py | 2 ++ api/messages.py | 2 ++ api/models.py | 2 ++ api/publicapi.py | 10 ++++-- api/redirect.py | 2 ++ api/searchutils.py | 82 ++++++++++++++++++++++--------------------- api/ticktockapi.py | 2 ++ 12 files changed, 96 insertions(+), 58 deletions(-) diff --git a/api/__init__.py b/api/__init__.py index f3b94a8..0e14f78 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -1,5 +1,7 @@ """The endpoints server.""" +from __future__ import division, print_function + import endpoints from calendarsapi import CalendarsAPI diff --git a/api/authutils.py b/api/authutils.py index 1368655..aaecfd2 100644 --- a/api/authutils.py +++ b/api/authutils.py @@ -6,6 +6,8 @@ subject to copyright by Google. """ +from __future__ import division, print_function + import os import json diff --git a/api/calendarsapi.py b/api/calendarsapi.py index 613aa3b..1441927 100644 --- a/api/calendarsapi.py +++ b/api/calendarsapi.py @@ -1,5 +1,7 @@ """API for managing calendars.""" +from __future__ import division, print_function + import logging import endpoints diff --git a/api/eventsapi.py b/api/eventsapi.py index f9409a1..23bad3d 100644 --- a/api/eventsapi.py +++ b/api/eventsapi.py @@ -1,5 +1,7 @@ """API for managing events.""" +from __future__ import division, print_function + import logging import endpoints @@ -112,13 +114,6 @@ def filter_and_update_events(unfiltered_events, starred_event_ids, chosen.append(event) return chosen - @staticmethod - def sort_and_search(events, search): - if search: - return searchutils.event_keyword_chron_search(events, search) - else: - return searchutils.event_chron_sort(events) - @endpoints.method(messages.EVENT_SEARCH_RESOURCE, messages.EventCollection, http_method="GET", path="/calendars/{calendarId}/events") def list(self, request): @@ -184,25 +179,44 @@ def list(self, request): else: extra_starred_ids = [] - # Initial sort and search, before checking the length - events = self.sort_and_search(starred_events, request.search) + events = starred_events[:] """:type: list[messages.EventProperties]""" + if request.search: + # Initial search, before checking the length + events = searchutils.event_keyword_search(events, request.search) + events += cached_events - # TODO: make this a for loop to 10, and last add search/sort to it - if len(events) < request.maxResults: + # TODO: handle excluded recurring events better + for _ in range(10): + if len(events) >= request.maxResults: + break + # Get event list from the google api api_events, gapi_next_page_token = gapiutils.get_events( service, request.calendarId, request.timeZone, gapi_next_page_token, request.maxResults) - events += self.filter_and_update_events( + api_events = self.filter_and_update_events( api_events, starred_event_ids, calendar_key, request.hidden) - # Sort and search again after adding api events - events = self.sort_and_search(events, request.search) - # TODO: if gapi_next_page_token is None: break + if request.search: + # Search again after adding more api events + api_events = searchutils.event_keyword_search(api_events, + request.search) + + events += api_events + + if gapi_next_page_token is None: + break + + # Do a final sort after adding all api events + if request.search: + events = searchutils.event_keyword_chron_sort(events, + request.search) + else: + events = searchutils.event_chron_sort(events) if len(events) >= request.maxResults: # Save extra for later diff --git a/api/gapiutils.py b/api/gapiutils.py index 1a7da1f..8d2241a 100644 --- a/api/gapiutils.py +++ b/api/gapiutils.py @@ -1,5 +1,7 @@ """Tools for getting data from the Google Calendar API.""" +from __future__ import division, print_function + import httplib from datetime import datetime, tzinfo diff --git a/api/garbagecollect.py b/api/garbagecollect.py index 3388105..ed9e7f6 100644 --- a/api/garbagecollect.py +++ b/api/garbagecollect.py @@ -1,5 +1,7 @@ """Clear out old or unbound datastore entities.""" +from __future__ import division, print_function + import logging from google.appengine.ext import ndb diff --git a/api/messages.py b/api/messages.py index b0a379f..f457bbb 100644 --- a/api/messages.py +++ b/api/messages.py @@ -1,5 +1,7 @@ """Endpoints messages.""" +from __future__ import division, print_function + from datetime import datetime from protorpc import messages, message_types diff --git a/api/models.py b/api/models.py index 31a5294..d755f10 100644 --- a/api/models.py +++ b/api/models.py @@ -1,5 +1,7 @@ """Datastore models.""" +from __future__ import division, print_function + import hashlib from google.appengine.ext import ndb diff --git a/api/publicapi.py b/api/publicapi.py index 245c4e9..ead5bfd 100644 --- a/api/publicapi.py +++ b/api/publicapi.py @@ -1,5 +1,7 @@ """API for accessing public calendars and events.""" +from __future__ import division, print_function + import endpoints from protorpc import remote from oauth2client.appengine import AppAssertionCredentials @@ -64,13 +66,15 @@ def events_list(self, request): authutils.CALENDAR_API_VERSION, AppAssertionCredentials(authutils.SERVICE_ACCOUNT_SCOPES) ) - events = gapiutils.get_events(service, request.calendarId, - request.timeZone, request.pageToken) + events, next_page_token = gapiutils.get_events( + service, request.calendarId, request.timeZone, + request.pageToken, request.maxResults) # Sort and search search = request.search if search: - events = searchutils.event_keyword_chron_search(events, search) + events = searchutils.event_keyword_search(events, search) + events = searchutils.event_keyword_chron_sort(events, search) else: events = searchutils.event_chron_sort(events) diff --git a/api/redirect.py b/api/redirect.py index 2c93ab2..5597757 100644 --- a/api/redirect.py +++ b/api/redirect.py @@ -1,5 +1,7 @@ """The endpoints server.""" +from __future__ import division, print_function + import webapp2 __author__ = "Alexander Otavka" diff --git a/api/searchutils.py b/api/searchutils.py index 51c46c4..37d3ddf 100644 --- a/api/searchutils.py +++ b/api/searchutils.py @@ -1,5 +1,7 @@ """Tools to help with searching and sorting API data.""" +from __future__ import division, print_function + from messages import EventProperties, CalendarProperties __author__ = "Alexander Otavka" @@ -12,31 +14,31 @@ def __init__(self): "Insufficient matches found for data item.") -def _get_event_kw_score(event, keywords, narrow=False): +def _get_event_kw_score(event, keywords, narrow): """ Get a relevance score for an event based on keyword matches. - :param EventProperties event: Event to be scored. - :param str keywords: Search terms separated by spaces. - :param bool narrow: - If true, throw NullSearchError for insufficient keyword matches. - + :type event: EventProperties + :type keywords: str + :param bool narrow: If true, throw NullSearchError for insufficient keyword + matches. :rtype: float - :raise NullSearchError: - If narrow=True and insufficient keyword matches are found. + :raise NullSearchError: If narrow is True and insufficient keyword matches + are found. """ - event_string_data = event.name + # TODO: make search algorithm less bad + event_string_data = event.name.lower() search_set = set(keywords.split()) matches = 0.0 for keyword in search_set: - if keyword in event_string_data: + if keyword.lower() in event_string_data: matches += 1.0 - if narrow and matches < len(search_set) // 2: + if narrow and (not matches or matches < len(search_set) // 2): raise NullSearchError() return matches -def _get_calendar_kw_score(calendar, keywords, narrow=False): +def _get_calendar_kw_score(calendar, keywords, narrow): """ Get a relevance score for a calendar based on keyword matches. @@ -98,24 +100,6 @@ def calendar_id_score(c): return c.calendarId -def event_chronological_order(): - return [event_starred, event_start_date, event_alpha_score, event_id_score] - - -def event_kw_chron_order(kw, narrow): - return [event_starred, event_kw_score(kw, narrow), event_start_date, - event_alpha_score, event_id_score] - - -def calendar_kw_alpha_order(kw, narrow): - return [calendar_kw_score(kw, narrow), calendar_alpha_score, - calendar_id_score] - - -def calendar_alpha_order(): - return [calendar_alpha_score, calendar_id_score] - - def search(search_list, order): """ Search and sort search_list based on tuple of order functions. @@ -134,46 +118,64 @@ def search(search_list, order): except NullSearchError: continue sorted_list = sorted(sorted_list) - return list(zip(*sorted_list)[-1]) + if sorted_list: + return list(zip(*sorted_list)[-1]) + else: + return [] + + +def event_keyword_search(event_list, keywords): + """ + Search exclusively by keyword order, and narrow results. + + :type event_list: list[EventProperties] + :type keywords: str + :rtype: list[EventProperties] + """ + return search(event_list, [event_kw_score(keywords, True)]) -def event_keyword_chron_search(event_list, keywords): +def event_keyword_chron_sort(event_list, keywords): """ - Convenience function, search with event_kw_chron_order. + Sort by keyword matches, then by start date, putting starred first. :type event_list: list[EventProperties] :type keywords: str :rtype: list[EventProperties] """ - return search(event_list, event_kw_chron_order(keywords, True)) + return search(event_list, [event_starred, event_kw_score(keywords, False), + event_start_date, event_alpha_score, + event_id_score]) def event_chron_sort(event_list): """ - Convenience function, search with event_chronological_order. + Sort events in chronological order, starred first. :type event_list: list[EventProperties] :rtype: list[EventProperties] """ - return search(event_list, event_chronological_order()) + return search(event_list, [event_starred, event_start_date, + event_alpha_score, event_id_score]) def calendar_keyword_alpha_search(calendar_list, keywords): """ - Convenience function, search with calendar_kw_alpha_order. + Search and narrow by keyword matches, then alphabetical order. :type calendar_list: list[CalendarProperties] :type keywords: str :rtype: list[CalendarProperties] """ - return search(calendar_list, calendar_kw_alpha_order(keywords, True)) + return search(calendar_list, [calendar_kw_score(keywords, True), + calendar_alpha_score, calendar_id_score]) def calendar_alpha_sort(calendar_list): """ - Convenience function, search with calendar_alpha_order. + Sort calendars in alphabetical order. :type calendar_list: list[CalendarProperties] :rtype: list[CalendarProperties] """ - return search(calendar_list, calendar_alpha_order()) + return search(calendar_list, [calendar_alpha_score, calendar_id_score]) diff --git a/api/ticktockapi.py b/api/ticktockapi.py index dbfedf0..73de4ef 100644 --- a/api/ticktockapi.py +++ b/api/ticktockapi.py @@ -1,5 +1,7 @@ """TickTock API definition.""" +from __future__ import division, print_function + import endpoints __author__ = "Alexander Otavka"