fixes #42 to freeze tw_cat

unfoldingWord-dev · Dec 13, 2017 · 557deec · 557deec
1 parent 7924ea5
commit 557deec
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 20 deletions.
diff --git a/libraries/lambda_handlers/ts_v2_catalog_handler.py b/libraries/lambda_handlers/ts_v2_catalog_handler.py
@@ -135,7 +135,6 @@ def __execute(self):
                             if process_id not in self.status['processed']:
                                 (tn, tw_cat) = self._index_note_files(lid, rid, format, process_id)
                                 if tn or tw_cat:
-                                    self._upload_all(tw_cat)
                                     self._upload_all(tn)
                                     finished_processes[process_id] = tw_cat.keys() + tn.keys()
                                     cat_keys = cat_keys + tn.keys() + tw_cat.keys()
@@ -196,7 +195,6 @@ def __execute(self):
                             if process_id not in self.status['processed']:
                                 (tn, tw_cat) = self._index_note_files(lid, rid, format, process_id)
                                 if tn or tw_cat:
-                                    self._upload_all(tw_cat)
                                     self._upload_all(tn)
                                     finished_processes[process_id] = tn.keys() + tw_cat.keys()
                                     cat_keys = cat_keys + tn.keys() + tw_cat.keys()
@@ -428,12 +426,8 @@ def _index_note_files(self, lid, rid, format, process_id):
 
                 if tw_chapters:
                     tw_cat_key = '_'.join([lid, '*', pid, 'tw'])
-                    tw_cat_json = {
-                        "chapters": tw_chapters,
-                        "date_modified": dc['modified'].replace('-', '')
-                    }
-                    tw_upload = self._prep_data_upload('{}/{}/tw_cat.json'.format(pid, lid), tw_cat_json)
-                    tw_cat_uploads[tw_cat_key] = tw_upload
+                    # TRICKY: we are freezing tw_cat but we still need the keys
+                    tw_cat_uploads[tw_cat_key] = ''
 
                 if note_json:
                     tn_key = '_'.join([lid, '*', pid, 'tn'])

diff --git a/tests/maptwtousfm3/maptwtousfm3_test.py b/tests/maptwtousfm3/maptwtousfm3_test.py
@@ -106,8 +106,6 @@ def test_map_dir(self):
         self.assertEqual(mapped_usfm, expected_usfm)
 
     def test_map_phrases(self):
-        rc = factory.load(os.path.join(self.resources_dir, 'tw_rc'))
-        strongs_index = maptwtousfm3.indexWordByStrongs(rc)
         usfm = read_file(os.path.join(self.resources_dir, 'mapped_tit.usfm'))
         mapped_usfm = maptwtousfm3.mapPhrases(usfm)
         expected_usfm = read_file(os.path.join(self.resources_dir, 'mapped_phrases_tit.usfm'))

diff --git a/tests/maptwtousfm3/resources/mapped_phrases_tit.usfm b/tests/maptwtousfm3/resources/mapped_phrases_tit.usfm
@@ -18,7 +18,7 @@
 \w δὲ|lemma="δέ" strong="G11610" x-morph="Gr,CC,,,,,,,,"\w*
 \k-s | x-tw="rc://*/tw/dict/bible/kt/jesus"
 \w Ἰησοῦ|lemma="Ἰησοῦς" strong="G24240" x-morph="Gr,N,,,,,GMS," \w*
-\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," \w*
+\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," x-tw="rc://*/tw/dict/bible/kt/christ"  \w*
 \k-e\*,
 \w κατὰ|lemma="κατά" strong="G25960" x-morph="Gr,P,,,,,A,,,"\w*
 \w πίστιν|lemma="πίστις" strong="G41020" x-morph="Gr,N,,,,,AFS,"\w*
@@ -80,7 +80,7 @@
 \w Πατρὸς|lemma="πατήρ" strong="G39620" x-morph="Gr,N,,,,,GMS,"\w*
 \w καὶ|lemma="καί" strong="G25320" x-morph="Gr,CC,,,,,,,,"\w*
 \k-s | x-tw="rc://*/tw/dict/bible/kt/jesus"
-\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," \w*
+\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," x-tw="rc://*/tw/dict/bible/kt/christ"  \w*
 \w Ἰησοῦ|lemma="Ἰησοῦς" strong="G24240" x-morph="Gr,N,,,,,GMS," \w*
 \k-e\*
 \w τοῦ|lemma="ὁ" strong="G35880" x-morph="Gr,EA,,,,GMS,"\w*

diff --git a/tests/tools_tests/resources/usfm_milestone.usfm b/tests/tools_tests/resources/usfm_milestone.usfm
@@ -1,4 +1,4 @@
 \k-s | x-tw="rc://*/tw/dict/bible/kt/jesus"
 \w Ἰησοῦ|lemma="Ἰησοῦς" strong="G24240" x-morph="Gr,N,,,,,GMS," \w*
-\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," \w*
+\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," x-tw="rc://*/tw/dict/bible/kt/christ"  \w*
 \k-e\*,
diff --git a/tests/tools_tests/test_tools.py b/tests/tools_tests/test_tools.py
@@ -103,11 +103,11 @@ def test_tw_phrase_add(self):
 
     def test_tw_phrase_print(self):
         phrase = tWPhrase(1)
-        phrase.addLine('\w Ἰησοῦ|lemma="Ἰησοῦς" strong="G24240" x-morph="Gr,N,,,,,GMS," x-tw="rc://*/tw/dict/bible/kt/jesus" \w*')
-        phrase.addLine('\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," x-tw="rc://*/tw/dict/bible/kt/christ"  x-tw="rc://*/tw/dict/bible/kt/jesus" \w*,')
+        phrase.addLine(u'\w Ἰησοῦ|lemma="Ἰησοῦς" strong="G24240" x-morph="Gr,N,,,,,GMS," x-tw="rc://*/tw/dict/bible/kt/jesus" \w*')
+        phrase.addLine(u'\w Χριστοῦ|lemma="χριστός" strong="G55470" x-morph="Gr,N,,,,,GMS," x-tw="rc://*/tw/dict/bible/kt/christ"  x-tw="rc://*/tw/dict/bible/kt/jesus" \w*,')
 
         expected = read_file(os.path.join(self.resources_dir, 'usfm_milestone.usfm'))
-        self.assertEqual(expected, unicode(str(phrase), 'utf-8'))
+        self.assertEqual(unicode(expected), unicode(phrase))
 
     @unittest.skipIf(is_travis(), 'Skipping test_is_lambda_not_running on travis')
     def test_is_lambda_is_not_running(self):

diff --git a/tests/ts_v2_catalog/test_ts_catalog.py b/tests/ts_v2_catalog/test_ts_catalog.py
@@ -60,7 +60,8 @@ def test_inprogress(self, mock_reporter):
         # assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/obs/source.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/notes.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/questions.json')
-        assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/tw_cat.json')
+        # we have frozen tw_cat
+        self.assertNotIn('v2/ts/obs/en/tw_cat.json', mockS3._recent_uploads)
 
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/languages.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/resources.json')
@@ -131,14 +132,16 @@ def test_convert_catalog(self, mock_reporter):
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/obs/v4/source.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/notes.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/questions.json')
-        assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/tw_cat.json')
+        # we have frozen tw_cat
+        self.assertNotIn('v2/ts/obs/en/tw_cat.json', mockS3._recent_uploads)
 
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/languages.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/resources.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/ulb/v7/source.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/notes.json')
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/questions.json')
-        assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/tw_cat.json')
+        # we have frozen tw_cat
+        self.assertNotIn('v2/ts/1ch/en/tw_cat.json', mockS3._recent_uploads)
 
         assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/bible/en/words.json')
 
@@ -176,7 +179,8 @@ def test_convert_catalog(self, mock_reporter):
                     if terms_path:
                         self.assertIn(terms_path, mockS3._recent_uploads, url_err_msg.format(terms_path))
                     if terms_map_path:
-                        self.assertIn(terms_map_path, mockS3._recent_uploads, url_err_msg.format(terms_map_path))
+                        # we have frozen tw_cat
+                        self.assertNotIn(terms_map_path, mockS3._recent_uploads, url_err_msg.format(terms_map_path))
 
     def test_complete_status(self, mock_reporter):
         mockV3Api = MockAPI(self.resources_dir, 'https://cdn.door43.org/')