replaces Lexicon (from ticcutils) by a 'normal' map<string,string>

LanguageMachines · Apr 4, 2020 · be97578 · be97578
1 parent f8717df
commit be97578
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 13 deletions.
diff --git a/include/mbt/Sentence.h b/include/mbt/Sentence.h
@@ -30,7 +30,6 @@
 #include "ticcutils/TreeHash.h"
 
 namespace Tagger {
-  using Hash::Lexicon;
   using Hash::StringHash;
 
   const std::string DOT = "==";
@@ -66,7 +65,7 @@ namespace Tagger {
     sentence( const PatTemplate&, const PatTemplate& );
     ~sentence();
     void clear();
-    bool init_windowing( Lexicon&, StringHash& );
+    bool init_windowing( std::map<std::string,std::string>&, StringHash& );
     bool nextpat( MatchAction&, std::vector<int>&, StringHash& , StringHash&,
 		  unsigned int, int * = 0 ) const;
     int classify_hapax( const std::string&, StringHash& ) const;

diff --git a/include/mbt/Tagger.h b/include/mbt/Tagger.h
@@ -212,8 +212,7 @@ namespace Tagger {
 
     PatTemplate Ktemplate;
     PatTemplate Utemplate;
-    Lexicon *MT_lexicon;
-
+    std::map<std::string,std::string> *MT_lexicon;
     std::string UnknownTreeBaseName;
     std::string KnownTreeBaseName;
     std::string LexFileBaseName;

diff --git a/src/GenerateTagger.cxx b/src/GenerateTagger.cxx
@@ -138,7 +138,7 @@ namespace Tagger {
       COUT << "  Creating ambitag lexicon: "  << MTLexFileName << endl;
       for ( const auto& tv : TagVect ){
 	out_file << tv->Word << " " << tv->stringRep() << endl;
-	MT_lexicon->Store( tv->Word, tv->stringRep() );
+	MT_lexicon->insert( make_pair(tv->Word, tv->stringRep() ) );
       }
       out_file.close();
     }

diff --git a/src/RunTagger.cxx b/src/RunTagger.cxx
@@ -427,7 +427,7 @@ namespace Tagger {
     int no_words=0;
     ifstream lexfile( FileName, ios::in);
     while ( lexfile >> wordbuf >> valbuf ){
-      MT_lexicon->Store( wordbuf, valbuf );
+      MT_lexicon->insert(make_pair(wordbuf,valbuf));
       no_words++;
       lexfile >> ws;
     }

diff --git a/src/Sentence.cxx b/src/Sentence.cxx
@@ -29,6 +29,7 @@
 #include <fstream>
 #include <iostream>
 #include <vector>
+#include <map>
 #include <string>
 #include <cstdlib>
 #include <cctype>
@@ -139,7 +140,7 @@ namespace Tagger {
     add(a_word, tmp, a_tag);
   }
 
-  bool sentence::init_windowing( Lexicon &lex,
+  bool sentence::init_windowing( map<string,string>& lex,
 				 StringHash& TheLex ) {
     if ( UTAG == -1 ){
 #pragma omp critical (hasher)
@@ -159,12 +160,11 @@ namespace Tagger {
 	}
 	// look up ambiguous tag in the dictionary
 	//
-	LexInfo *foundInfo = lex.Lookup( cur_word->the_word );
-	if ( foundInfo != NULL ){
-	  //	  cerr << "MT Lookup(" << cur_word->the_word << ") gave " << *foundInfo << endl;
+	const auto it = lex.find( cur_word->the_word );
+	if ( it != lex.end() ){
 #pragma omp critical (hasher)
 	  {
-	    cur_word->word_amb_tag = TheLex.Hash( foundInfo->Trans() );
+	    cur_word->word_amb_tag = TheLex.Hash( it->second );
 	  }
 	}
 	else  {

diff --git a/src/Tagger.cxx b/src/Tagger.cxx
@@ -95,7 +95,7 @@ namespace Tagger {
     initialized = false;
     Beam_Size = 1;
     Beam = NULL;
-    MT_lexicon = new Lexicon();
+    MT_lexicon = new map<string,string>;
     kwordlist = new StringHash();
     uwordlist = new StringHash();
     piped_input = true;
@@ -155,7 +155,7 @@ namespace Tagger {
     Separators( in.Separators ),
     Ktemplate( in.Ktemplate ),
     Utemplate( in.Utemplate ),
-    MT_lexicon( in.MT_lexicon ),
+    MT_lexicon( in.MT_lexicon ),     //!> is a pointer to avoid copies
     UnknownTreeBaseName( in.UnknownTreeBaseName ),
     KnownTreeBaseName( in.KnownTreeBaseName ),
     LexFileBaseName( in.LexFileBaseName ),