104 fixes (#527)

* added stemmed synonym searching to synonym field in ms Signed-off-by: Kial Jinnah <[email protected]> * pytest added Signed-off-by: Kial Jinnah <[email protected]> * pytest update Signed-off-by: Kial Jinnah <[email protected]>
bcgov · Feb 20, 2019 · 59bba39 · 59bba39
1 parent a110c7d
commit 59bba39
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 7 deletions.
diff --git a/api/tests/python/end_points/test_synonym_match.py b/api/tests/python/end_points/test_synonym_match.py
@@ -550,3 +550,20 @@ def test_number_synonyms(client, jwt, app, query, expected_list):
     seed_database_with(client, jwt, 'FIFTH ZZZZZ', id='1', source='2')
     verify_synonym_match(client, jwt, query=query, expected_list=expected_list)
 
+@integration_postgres_solr
+@integration_synonym_api
+@integration_solr
+@pytest.mark.parametrize("query, ordered_list", [
+    ('PACIFIC WEST CONSTRUCTION', ['----PACIFIC WEST CONSTRUCTION - PROXIMITY SEARCH',
+                                    '----PACIFIC WEST CONSTRUCTION* - EXACT WORD ORDER',
+                                    '----PACIFIC WEST synonyms:(CONSTRUCT) - PROXIMITY SEARCH',
+                                    '----PACIFIC WEST* synonyms:(CONSTRUCT) - EXACT WORD ORDER',
+                                    '----PACIFIC synonyms:(CONSTRUCT) - PROXIMITY SEARCH',
+                                    'PACIFIC DEVELOPMENT',
+                                                  ]),
+])
+def test_synonym_clause_stemmed(client, jwt, app, query, ordered_list):
+    #  for loop didn't work for seeding so manual
+    seed_database_with(client, jwt, 'PACIFIC DEVELOPMENT', id='1', source='2')
+    verify_order(client, jwt, query=query, expected_order=ordered_list)
+
diff --git a/solr/cores/possible.conflicts/conf/managed-schema b/solr/cores/possible.conflicts/conf/managed-schema
@@ -687,20 +687,18 @@
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
         <filter class="solr.StopFilterFactory" words="lang/stopwords_en.txt" ignoreCase="true"/>
         <filter class="solr.LowerCaseFilterFactory"/>
-        <filter class="solr.PorterStemFilterFactory"/>
-        <filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="0"
-            splitOnCaseChange="0" generateWordParts="0" catenateAll="1" catenateWords="1"/>
+        <filter class="solr.SnowballPorterFilterFactory" protected="protected_syns.txt" language="English"/>
         <filter class="solr.FlattenGraphFilterFactory"/>
         <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
       </analyzer>
       <analyzer type="query">
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
         <filter class="solr.StopFilterFactory" words="lang/stopwords_en.txt" ignoreCase="true"/>
         <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.SnowballPorterFilterFactory" protected="protected_syns.txt" language="English"/>
 		<filter class="com.s24.search.solr.analysis.jdbc.JdbcSynonymFilterFactory" dataSource="jdbc/synonyms"
-            sql="select synonyms_text from synonym where enabled = true" ignoreMissingDatabase="true"
+            sql="select stems_text from synonym where enabled = true" ignoreMissingDatabase="true"
             ignoreCase="true" expand="true"/>
-        <filter class="solr.PorterStemFilterFactory"/>
         <filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="0" splitOnCaseChange="0" generateWordParts="0" catenateAll="1" catenateWords="1"/>
         <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
         <filter class="solr.FlattenGraphFilterFactory"/>
@@ -851,7 +849,6 @@
                 ignoreCase="true"
                 words="lang/stopwords_en.txt"
         />
-        <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.FlattenGraphFilterFactory"/>
       </analyzer>
@@ -880,7 +877,6 @@
                 ignoreCase="true"
                 words="lang/stopwords_en.txt"
         />
-        <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.FlattenGraphFilterFactory"/>
       </analyzer>