From d9df457992a8bf81058bff2fb0625f860b1f85b2 Mon Sep 17 00:00:00 2001 From: Wei Zhuo Date: Fri, 18 Sep 2015 14:09:55 +1000 Subject: [PATCH] Use db for caching models --- .../main/java/io/github/xwz/base/Utils.java | 13 +- .../{content => api}/ContentDatabase.java | 2 +- .../xwz/base/api/ContentDatabaseCache.java | 178 ++++++++++++++++++ .../github/xwz/base/api/EpisodeBaseModel.java | 40 +++- .../io/github/xwz/base/api/HttpApiBase.java | 32 ---- .../xwz/base/content/ContentCacheManager.java | 43 ++++- .../base/content/ContentDatabaseCache.java | 41 ---- .../xwz/base/content/ContentManagerBase.java | 7 + .../io/github/xwz/iview/MainApplication.java | 2 +- .../github/xwz/iview/api/TvShowListApi.java | 20 +- .../io/github/xwz/sbs/api/EpisodeModel.java | 1 - .../java/io/github/xwz/sbs/api/SBSApi.java | 18 +- .../io/github/xwz/sbs/api/SBSApiBase.java | 3 - 13 files changed, 301 insertions(+), 99 deletions(-) rename base/src/main/java/io/github/xwz/base/{content => api}/ContentDatabase.java (87%) create mode 100644 base/src/main/java/io/github/xwz/base/api/ContentDatabaseCache.java delete mode 100644 base/src/main/java/io/github/xwz/base/content/ContentDatabaseCache.java diff --git a/base/src/main/java/io/github/xwz/base/Utils.java b/base/src/main/java/io/github/xwz/base/Utils.java index e059ab7..e903f3f 100644 --- a/base/src/main/java/io/github/xwz/base/Utils.java +++ b/base/src/main/java/io/github/xwz/base/Utils.java @@ -73,11 +73,14 @@ public static String formatMillis(int millis) { } public static String stripCategory(String str) { - String[] parts = str.split("/"); - if (parts.length > 1) { - return parts[1]; - } else { - return str; + if (str != null) { + String[] parts = str.split("/"); + if (parts.length > 1) { + return parts[1]; + } else { + return str; + } } + return ""; } } \ No newline at end of file diff --git a/base/src/main/java/io/github/xwz/base/content/ContentDatabase.java b/base/src/main/java/io/github/xwz/base/api/ContentDatabase.java similarity index 87% rename from base/src/main/java/io/github/xwz/base/content/ContentDatabase.java rename to base/src/main/java/io/github/xwz/base/api/ContentDatabase.java index f668adf..bb57460 100644 --- a/base/src/main/java/io/github/xwz/base/content/ContentDatabase.java +++ b/base/src/main/java/io/github/xwz/base/api/ContentDatabase.java @@ -1,4 +1,4 @@ -package io.github.xwz.base.content; +package io.github.xwz.base.api; import com.raizlabs.android.dbflow.annotation.Database; diff --git a/base/src/main/java/io/github/xwz/base/api/ContentDatabaseCache.java b/base/src/main/java/io/github/xwz/base/api/ContentDatabaseCache.java new file mode 100644 index 0000000..40e1522 --- /dev/null +++ b/base/src/main/java/io/github/xwz/base/api/ContentDatabaseCache.java @@ -0,0 +1,178 @@ +package io.github.xwz.base.api; + +import android.util.Log; + +import com.raizlabs.android.dbflow.list.FlowCursorList; +import com.raizlabs.android.dbflow.list.FlowQueryList; +import com.raizlabs.android.dbflow.sql.builder.Condition; +import com.raizlabs.android.dbflow.sql.language.Select; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import io.github.xwz.base.content.ContentCacheManager; +import io.github.xwz.base.content.ContentManagerBase; + +public class ContentDatabaseCache { + private static final String TAG = "ContentDatabaseCache"; + + private static final String TYPE_EPISODES = "EPISODES"; + private static final String TYPE_COLLECTIONS = "COLLECTIONS"; + private static final String TYPE_SHOWS = "SHOWS"; + + public void clearCache() { + Log.d(TAG, "Clear db"); + FlowQueryList query = new FlowQueryList<>(EpisodeBaseModel.class); + query.clear(); + } + + public void putShows(Collection shows) { + Log.d(TAG, "store shows into db"); + FlowQueryList query = new FlowQueryList<>(EpisodeBaseModel.class); + query.beginTransaction(); + for (EpisodeBaseModel ep : shows) { + EpisodeBaseModel model = new EpisodeBaseModel(); + model.merge(ep); + model.DATA_TYPE = TYPE_EPISODES; + model.save(); + } + query.endTransactionAndNotify(); + } + + public void putCollections(LinkedHashMap> collections) { + Log.d(TAG, "store collections into db"); + int i = 0; + FlowQueryList query = new FlowQueryList<>(EpisodeBaseModel.class); + query.beginTransaction(); + for (Map.Entry> collection : collections.entrySet()) { + Log.d(TAG, "Adding collection: " + collection.getKey() + " => " + collection.getValue().size()); + updateProgress("Loading " + collection.getKey() + "..."); + for (EpisodeBaseModel ep : collection.getValue()) { + EpisodeBaseModel model = new EpisodeBaseModel(); + model.merge(ep); + model.DATA_TYPE = TYPE_COLLECTIONS; + model.DATA_COLLECTION_KEY = collection.getKey(); + model.DATA_COLLECTION_INDEX = i++; + model.save(); + } + } + query.endTransactionAndNotify(); + } + + private void updateProgress(String str) { + ContentManagerBase.getInstance().broadcastChange(ContentManagerBase.CONTENT_SHOW_LIST_PROGRESS, str); + } + + public void putEpisodes(Collection episodes) { + Log.d(TAG, "store episodes into db"); + FlowQueryList query = new FlowQueryList<>(EpisodeBaseModel.class); + query.beginTransaction(); + for (EpisodeBaseModel ep : episodes) { + EpisodeBaseModel model = new EpisodeBaseModel(); + model.merge(ep); + model.DATA_TYPE = TYPE_SHOWS; + model.save(); + } + query.endTransactionAndNotify(); + } + + private List getModelsOfType(Class model, String type, List existing, boolean uniqueSeries) { + FlowCursorList cursor = new FlowCursorList<>(false, EpisodeBaseModel.class, + Condition.column(EpisodeBaseModel$Table.DATA_TYPE).eq(type)); + Map all = new HashMap<>(); + for (int i = 0, k = cursor.getCount(); i < k; i++) { + EpisodeBaseModel ep = (EpisodeBaseModel) createInstanceOf(model); + if (ep != null) { + int index = existing.indexOf(ep); + if (index > -1) { + ep = existing.get(index); + } else { + EpisodeBaseModel item = cursor.getItem(i); + item.unserialize(); + ep.merge(item); + } + if (uniqueSeries) { + all.put(ep.getSeriesTitle(), ep); + } else { + all.put(ep.getHref(), ep); + } + } + } + cursor.close(); + return new ArrayList<>(all.values()); + } + + private Object createInstanceOf(Class type) { + try { + Constructor ctor = type.getConstructor(); + Object object = ctor.newInstance(); + return object; + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } catch (InstantiationException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + return null; + } + + private LinkedHashMap> getCollections(Class model, List existing) { + LinkedHashMap> collections = new LinkedHashMap<>(); + FlowCursorList cursor = new FlowCursorList<>(false, + (new Select()).from(EpisodeBaseModel.class) + .where(Condition.column(EpisodeBaseModel$Table.DATA_TYPE).eq(TYPE_COLLECTIONS)) + .orderBy(true, EpisodeBaseModel$Table.DATA_COLLECTION_INDEX)); + for (int i = 0, k = cursor.getCount(); i < k; i++) { + EpisodeBaseModel item = cursor.getItem(i); + item.unserialize(); + int index = existing.indexOf(item); + EpisodeBaseModel ep; + if (index > -1) { + ep = existing.get(index); + } else { + ep = (EpisodeBaseModel) createInstanceOf(model); + if (ep != null) { + ep.merge(item); + } + } + if (ep != null) { + if (!collections.containsKey(item.DATA_COLLECTION_KEY)) { + collections.put(item.DATA_COLLECTION_KEY, new ArrayList()); + } + collections.get(item.DATA_COLLECTION_KEY).add(ep); + } + } + for (Map.Entry> collection : collections.entrySet()) { + Log.d(TAG, "Loaded collection: " + collection.getKey() + " => " + collection.getValue().size()); + } + return collections; + } + + public boolean loadFromDbCache(ContentCacheManager cache, Class type) { + updateProgress("Loading images..."); + List episodes = getModelsOfType(type, TYPE_EPISODES, new ArrayList(), false); + if (episodes.size() > 0) { + updateProgress("Loading TV shows..."); + List shows = getModelsOfType(type, TYPE_SHOWS, episodes, true); + updateProgress("Loading movies..."); + LinkedHashMap> collections = getCollections(type, episodes); + cache.putEpisodes(episodes); + cache.putShows(shows); + cache.putCollections(collections); + updateProgress("Loading content..."); + cache.buildDictionary(shows); + Log.d(TAG, "Loaded data from database"); + return true; + } + return false; + } +} diff --git a/base/src/main/java/io/github/xwz/base/api/EpisodeBaseModel.java b/base/src/main/java/io/github/xwz/base/api/EpisodeBaseModel.java index 315baec..de4644e 100644 --- a/base/src/main/java/io/github/xwz/base/api/EpisodeBaseModel.java +++ b/base/src/main/java/io/github/xwz/base/api/EpisodeBaseModel.java @@ -7,6 +7,9 @@ import com.raizlabs.android.dbflow.annotation.Table; import com.raizlabs.android.dbflow.structure.BaseModel; +import org.json.JSONArray; +import org.json.JSONException; + import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; @@ -17,17 +20,17 @@ import java.util.Set; import io.github.xwz.base.Utils; -import io.github.xwz.base.content.ContentDatabase; @Table(databaseName = ContentDatabase.NAME) public class EpisodeBaseModel extends BaseModel implements Serializable { + private static final String TAG = "EpisodeBaseModel"; + @Column - @PrimaryKey - private String href; + @PrimaryKey(autoincrement = true) + public long DATA_ID; @Column - @PrimaryKey public String DATA_TYPE; @Column @@ -36,6 +39,9 @@ public class EpisodeBaseModel extends BaseModel implements Serializable { @Column public int DATA_COLLECTION_INDEX; + @Column + private String href; + @Column private String seriesTitle; @@ -101,6 +107,32 @@ public void update() { super.update(); } + void unserialize() { + if (categoriesSerialized != null) { + JSONArray arr = parseArray(categoriesSerialized); + if (arr != null) { + for (int i = 0; i < arr.length();i++) { + try { + addCategory(arr.getString(i)); + } catch (JSONException e) { + e.printStackTrace(); + } + } + } + } + } + + private JSONArray parseArray(String content) { + if (content != null && content.contains("[") && content.contains("]")) { + try { + return new JSONArray(content); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return null; + } + public void addCategory(String cat) { categories.add(cat); } diff --git a/base/src/main/java/io/github/xwz/base/api/HttpApiBase.java b/base/src/main/java/io/github/xwz/base/api/HttpApiBase.java index 1ae7e43..04a3bee 100644 --- a/base/src/main/java/io/github/xwz/base/api/HttpApiBase.java +++ b/base/src/main/java/io/github/xwz/base/api/HttpApiBase.java @@ -170,38 +170,6 @@ private static File createDefaultCacheDir(Context context, String path) { return cache; } - protected RadixTree buildWordsFromShows(Collection shows) { - RadixTree dict = new RadixTree<>(); - for (EpisodeBaseModel ep : shows) { - dict.putAll(getWords(ep)); - } - Log.d(TAG, "dict:" + dict.size()); - return dict; - } - - private Map getWords(EpisodeBaseModel episode) { - Map words = new HashMap<>(); - if (episode.getSeriesTitle() != null) { - words.putAll(splitWords(episode.getSeriesTitle(), episode)); - } - if (episode.getTitle() != null) { - words.putAll(splitWords(episode.getTitle(), episode)); - } - return words; - } - - private Map splitWords(String s, EpisodeBaseModel episode) { - String[] words = s.split("\\s+"); - Map result = new HashMap<>(); - for (String w : words) { - String word = w.replaceAll("[^\\w]", ""); - if (word.length() >= 3) { - result.put(word.toLowerCase(), word); - } - } - return result; - } - private static long calculateDiskCacheSize(File dir) { long size = Math.min(calculateAvailableCacheSize(dir), MAX_DISK_CACHE_SIZE); return Math.max(size, MIN_DISK_CACHE_SIZE); diff --git a/base/src/main/java/io/github/xwz/base/content/ContentCacheManager.java b/base/src/main/java/io/github/xwz/base/content/ContentCacheManager.java index bbe598e..cfb95ce 100644 --- a/base/src/main/java/io/github/xwz/base/content/ContentCacheManager.java +++ b/base/src/main/java/io/github/xwz/base/content/ContentCacheManager.java @@ -21,7 +21,7 @@ public class ContentCacheManager { private static final String TAG = "ContentCacheManager"; private final LocalBroadcastManager mBroadcastManager; - private final Map mEpisodes = new HashMap<>(); + private Map mEpisodes = new HashMap<>(); private List mShows = new ArrayList<>(); private RadixTree mDictionary = new RadixTree<>(); private final Map mStreamUrls = new HashMap<>(); @@ -87,14 +87,19 @@ synchronized public LinkedHashMap> getCollections return new LinkedHashMap<>(mCollections); } + synchronized public void putEpisodes(Collection episodes) { + mEpisodes = new HashMap<>(); + addEpisodes(episodes); + } + synchronized public void addEpisodes(Collection episodes) { for (EpisodeBaseModel ep : episodes) { mEpisodes.put(ep.getHref(), ep); } } - synchronized public void setDictionary(RadixTree dict) { - mDictionary = dict; + synchronized public void buildDictionary(Collection shows) { + mDictionary = buildWordsFromShows(shows); } synchronized public List getSuggestions(String query) { @@ -124,4 +129,36 @@ synchronized public void putStreamUrl(String id, Uri url) { synchronized public Uri getEpisodeStreamUrl(String id) { return mStreamUrls.get(id); } + + private RadixTree buildWordsFromShows(Collection shows) { + RadixTree dict = new RadixTree<>(); + for (EpisodeBaseModel ep : shows) { + dict.putAll(getWords(ep)); + } + Log.d(TAG, "dict:" + dict.size()); + return dict; + } + + private Map getWords(EpisodeBaseModel episode) { + Map words = new HashMap<>(); + if (episode.getSeriesTitle() != null) { + words.putAll(splitWords(episode.getSeriesTitle(), episode)); + } + if (episode.getTitle() != null) { + words.putAll(splitWords(episode.getTitle(), episode)); + } + return words; + } + + private Map splitWords(String s, EpisodeBaseModel episode) { + String[] words = s.split("\\s+"); + Map result = new HashMap<>(); + for (String w : words) { + String word = w.replaceAll("[^\\w]", ""); + if (word.length() >= 3) { + result.put(word.toLowerCase(), word); + } + } + return result; + } } \ No newline at end of file diff --git a/base/src/main/java/io/github/xwz/base/content/ContentDatabaseCache.java b/base/src/main/java/io/github/xwz/base/content/ContentDatabaseCache.java deleted file mode 100644 index 4ddfa06..0000000 --- a/base/src/main/java/io/github/xwz/base/content/ContentDatabaseCache.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.github.xwz.base.content; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.List; - -import io.github.xwz.base.api.EpisodeBaseModel; - -public class ContentDatabaseCache { - private static final String TYPE_EPISODES = "EPISODES"; - private static final String TYPE_COLLECTIONS = "COLLECTIONS"; - private static final String TYPE_SHOWS = "SHOWS"; - - - public List getAllShows() { - return new ArrayList<>(); - } - - public boolean hasShows() { - return false; - } - - public void putShows(Collection shows) { - } - - public void putCollections(LinkedHashMap> shows) { - } - - public LinkedHashMap> getCollections() { - return new LinkedHashMap<>(); - } - - public void putEpisodes(Collection episodes) { - - } - - public List getEpisodes() { - return new ArrayList<>(); - } -} diff --git a/base/src/main/java/io/github/xwz/base/content/ContentManagerBase.java b/base/src/main/java/io/github/xwz/base/content/ContentManagerBase.java index 76c666f..4578a91 100644 --- a/base/src/main/java/io/github/xwz/base/content/ContentManagerBase.java +++ b/base/src/main/java/io/github/xwz/base/content/ContentManagerBase.java @@ -13,6 +13,7 @@ import java.util.LinkedHashMap; import java.util.List; +import io.github.xwz.base.api.ContentDatabaseCache; import io.github.xwz.base.api.EpisodeBaseModel; public abstract class ContentManagerBase { @@ -64,6 +65,7 @@ public abstract class ContentManagerBase { private Context mContext = null; private ContentCacheManager mCache = null; + private ContentDatabaseCache mDb = null; public enum RecommendationPosition { FIRST(0), SECOND(1); @@ -82,6 +84,7 @@ public ContentManagerBase(Context context) { instance = this; mContext = context; mCache = new ContentCacheManager(context); + mDb = new ContentDatabaseCache(); } public static ContentManagerBase getInstance() { @@ -92,6 +95,10 @@ public static ContentCacheManager cache() { return getInstance().mCache; } + public static ContentDatabaseCache db() { + return getInstance().mDb; + } + protected Context getContext() { return mContext; } diff --git a/iview/src/main/java/io/github/xwz/iview/MainApplication.java b/iview/src/main/java/io/github/xwz/iview/MainApplication.java index 206863c..ae167a6 100644 --- a/iview/src/main/java/io/github/xwz/iview/MainApplication.java +++ b/iview/src/main/java/io/github/xwz/iview/MainApplication.java @@ -13,8 +13,8 @@ public class MainApplication extends Application implements IApplication { @Override public void onCreate() { super.onCreate(); - new ContentManager(this).fetchShowList(true); FlowManager.init(this); + new ContentManager(this).fetchShowList(true); } @Override diff --git a/iview/src/main/java/io/github/xwz/iview/api/TvShowListApi.java b/iview/src/main/java/io/github/xwz/iview/api/TvShowListApi.java index b9e4ecf..445ee9a 100644 --- a/iview/src/main/java/io/github/xwz/iview/api/TvShowListApi.java +++ b/iview/src/main/java/io/github/xwz/iview/api/TvShowListApi.java @@ -4,7 +4,6 @@ import android.net.Uri; import android.text.TextUtils; import android.util.Log; -import android.view.ContextMenu; import org.json.JSONArray; import org.json.JSONException; @@ -44,6 +43,10 @@ public TvShowListApi(Context context) { @Override protected Void doInBackground(String... urls) { + if (ContentManager.db().loadFromDbCache(ContentManager.cache(), EpisodeModel.class)) { + ContentManager.getInstance().broadcastChange(ContentManager.CONTENT_SHOW_LIST_DONE); + } + updateProgress(); fetchTitlesFromCollection(); @@ -57,9 +60,16 @@ protected Void doInBackground(String... urls) { updateProgress(); ContentManager.cache().putShows(shows); - ContentManager.cache().addEpisodes(episodes.values()); + ContentManager.cache().putEpisodes(episodes.values()); ContentManager.cache().putCollections(collections); - ContentManager.cache().setDictionary(buildWordsFromShows(shows)); + ContentManager.cache().buildDictionary(shows); + + ContentManager.db().clearCache(); + ContentManager.db().putShows(shows); + updateProgress(); + ContentManager.db().putEpisodes(episodes.values()); + updateProgress(); + ContentManager.db().putCollections(collections); updateProgress(); success = true; @@ -174,9 +184,7 @@ protected void onPreExecute() { } protected void onPostExecute(Void v) { - if (success) { - ContentManager.getInstance().broadcastChange(ContentManager.CONTENT_SHOW_LIST_DONE); - } else { + if (!success) { ContentManager.getInstance().broadcastChange(ContentManager.CONTENT_SHOW_LIST_ERROR); } episodes.clear(); diff --git a/sbs/src/main/java/io/github/xwz/sbs/api/EpisodeModel.java b/sbs/src/main/java/io/github/xwz/sbs/api/EpisodeModel.java index afaf42a..999633a 100644 --- a/sbs/src/main/java/io/github/xwz/sbs/api/EpisodeModel.java +++ b/sbs/src/main/java/io/github/xwz/sbs/api/EpisodeModel.java @@ -28,7 +28,6 @@ private void set(SBSApi.Entry data) { setCover(data.getCover()); setCategories(data.getCategories()); setDescription(data.synopsis); - setExpiry(data.expiry); } public void setHasFetchedRelated(boolean fetched) { diff --git a/sbs/src/main/java/io/github/xwz/sbs/api/SBSApi.java b/sbs/src/main/java/io/github/xwz/sbs/api/SBSApi.java index 99aad1b..f0eacc7 100644 --- a/sbs/src/main/java/io/github/xwz/sbs/api/SBSApi.java +++ b/sbs/src/main/java/io/github/xwz/sbs/api/SBSApi.java @@ -45,6 +45,10 @@ public SBSApi(Context context) { @Override protected Void doInBackground(String... params) { + if (ContentManager.db().loadFromDbCache(ContentManager.cache(), EpisodeModel.class)) { + ContentManager.getInstance().broadcastChange(ContentManager.CONTENT_SHOW_LIST_DONE); + } + while (getLastEntryCount() == 0 || getLastEntryCount() == ITEMS_PER_PAGE) { updateProgress(); fetchAllTitles(page++); @@ -103,9 +107,19 @@ protected Void doInBackground(String... params) { } updateProgress(); ContentManager.cache().putShows(shows); - ContentManager.cache().addEpisodes(episodes); + ContentManager.cache().putEpisodes(episodes); ContentManager.cache().putCollections(sortedCollection); - ContentManager.cache().setDictionary(buildWordsFromShows(shows)); + ContentManager.cache().buildDictionary(shows); + + ContentManager.getInstance().broadcastChange(ContentManager.CONTENT_SHOW_LIST_DONE); + + ContentManager.db().clearCache(); + ContentManager.db().putShows(shows); + updateProgress(); + ContentManager.db().putEpisodes(episodes); + updateProgress(); + ContentManager.db().putCollections(sortedCollection); + updateProgress(); success = true; diff --git a/sbs/src/main/java/io/github/xwz/sbs/api/SBSApiBase.java b/sbs/src/main/java/io/github/xwz/sbs/api/SBSApiBase.java index 85b07a3..9bc5238 100644 --- a/sbs/src/main/java/io/github/xwz/sbs/api/SBSApiBase.java +++ b/sbs/src/main/java/io/github/xwz/sbs/api/SBSApiBase.java @@ -249,9 +249,6 @@ public static class Entry { @SerializedName("pl1$shortSynopsis") String synopsis; - @SerializedName("media$expirationDate") - int expiry; - @SerializedName("media$categories") private List categories;