update token lexer

CMakeLists.txt

@@ -1,45 +1,55 @@
 cmake_minimum_required(VERSION 3.28)
 project(Trc CXX)
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 
 
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_C_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
 
-add_library(modulefiles)
+set(CMAKE_CXX_STANDARD 20)
+SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin)
+set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin)
+link_directories("${PROJECT_SOURCE_DIR}/bin")
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror")
+
+#add_subdirectory(language)
+
+#add_compile_definitions("UNITTEST")
+#file(GLOB_RECURSE TEST_SOURCES "tests/unittest/*.cpp")
+#file(GLOB_RECURSE TEST_MODULE_SOURCES "tests/unittest/*.cppm")
+#add_library(testmodulefiles)
+#
+#target_sources(testmodulefiles
+#        PUBLIC
+#        FILE_SET CXX_MODULES FILES
+#        ${SOURCES}
+#        ${LANGUAGE_SOURCES}
+#        ${TEST_MODULE_SOURCES}
+#)
+#
+#add_executable(unittest
+#    ${CPP_SOURCES}
+#    ${TEST_SOURCES}
+#)
+#target_link_libraries(unittest testmodulefiles language)
 
-file(GLOB_RECURSE SOURCES "src/*.cppm")
-file(GLOB_RECURSE LANGUAGE_SOURCES "language/*.cppm")
-add_subdirectory("language")
-target_sources(modulefiles
-        PUBLIC
-        FILE_SET CXX_MODULES FILES
-        ${SOURCES}
-        ${LANGUAGE_SOURCES}
-)
+add_subdirectory("cmake_third_party/googletest")
+include_directories("cmake_third_party/googletest/googletest/include")
 
+file(GLOB_RECURSE CPPMSOURCES "*.cppm")
 file(GLOB_RECURSE CPP_SOURCES "src/*.cpp")
-add_executable(Trc ${CPP_SOURCES})
 
-target_link_libraries(Trc modulefiles language)
+message(${CPP_SOURCES})
 
-add_compile_definitions("UNITTEST")
-file(GLOB_RECURSE TEST_SOURCES "tests/unittest/*.cpp")
-file(GLOB_RECURSE TEST_MODULE_SOURCES "tests/unittest/*.cppm")
-add_library(testmodulefiles)
+add_library(foo)
 
-target_sources(testmodulefiles
+target_sources(foo
         PUBLIC
         FILE_SET CXX_MODULES FILES
-        ${SOURCES}
-        ${LANGUAGE_SOURCES}
-        ${TEST_MODULE_SOURCES}
+        ${CPPMSOURCES}
 )
 
-add_executable(unittest
-    ${CPP_SOURCES}
-    ${TEST_SOURCES}
-)
-target_link_libraries(Trc testmodulefiles language)
+add_executable(Trc ${CPP_SOURCES})
 
-add_subdirectory("cmake_third_party/googletest")
-include_directories("cmake_third_party/googletest/googletest/include")
+target_link_libraries(Trc foo)
+target_link_libraries(Trc "language")

language/CMakeLists.txt

@@ -1,24 +1,20 @@
-add_library(languagemodule)
 file(GLOB_RECURSE LANGUAGE_SOURCES "*.cppm")
 
-target_sources(languagemodule
+add_library(language_module)
+target_sources(language_module
         PUBLIC
         FILE_SET CXX_MODULES FILES
         ${LANGUAGE_SOURCES}
 )
 
-add_library(Chinese SHARED
-    chinese.cpp
-)
+message("${LANGUAGE_SOURCES}")
 
-add_library(English SHARED
-        english.cpp
-)
+add_library(Chinese chinese.cpp)
 
-add_library(language SHARED
-        english.cpp
-)
+add_library(English english.cpp)
+
+add_library(language english.cpp)
 
-target_link_libraries(Chinese languagemodule)
-target_link_libraries(English languagemodule)
-target_link_libraries(language languagemodule)
+target_link_libraries(Chinese language_module)
+target_link_libraries(English language_module)
+target_link_libraries(language language_module)

src/Compiler/Compiler.cppm

@@ -217,7 +217,7 @@ bool Compiler::id(bool error_report) {
     }
     return true;
 failed:
-    token_.unget_token(t);
+    token_.unget_token();
     return false;
 }
 
@@ -287,7 +287,7 @@ bool Compiler::const_value() {
         return true;
     }
     default: {
-        token_.unget_token(data_token);
+        token_.unget_token();
         return false;
     }
     }
@@ -445,7 +445,7 @@ bool Compiler::item(bool error_reoprt) {
 bool Compiler::match(token_ticks tick) {
     token res = token_.get_token();
     if (res.tick != tick) {
-        token_.unget_token(res);
+        token_.unget_token();
         return false;
     }
     return true;
@@ -454,7 +454,7 @@ bool Compiler::match(token_ticks tick) {
 token_ticks Compiler::get_next_token_tick() {
     token next_tmp = token_.get_token();
     auto nexttick = next_tmp.tick;
-    token_.unget_token(next_tmp);
+    token_.unget_token();
     return nexttick;
 }
 }

src/Compiler/token.cppm

@@ -110,16 +110,19 @@ public:
     /**
      * @brief 退回并储存一个token
      */
-    void unget_token(token token_data);
+    void unget_token();
 
     compiler_public_data& compiler_data;
 
 private:
-    token back_token;
-    bool has_back_token = false;
+    std::vector<token> tokenlist;
+
+    int id = 0;
 
     // 指向当前正在解析的字符
-    const char* char_ptr {};
+    const char* char_ptr = nullptr;
+
+    token lexinteral();
 
     // 判断是否解析到了终点
     [[nodiscard]] bool end_of_lex() const noexcept;
@@ -534,23 +537,27 @@ token token_lex::lex_others() {
     return result;
 }
 
-void token_lex::unget_token(token token_data) {
-    assert(!has_back_token);
-    if (token_data.tick == token_ticks::END_OF_LINE) {
+void token_lex::unget_token() {
+    if(id == 0) {
+        unreach("token list out of range");
+    }
+    id--;
+    if (tokenlist[id].tick == token_ticks::END_OF_LINE) {
         compiler_data.error.sub_line();
     }
-    back_token = token_data;
-    has_back_token = true;
 }
 
 token token_lex::get_token() {
-    if (has_back_token) {
-        has_back_token = false;
-        if (back_token.tick == token_ticks::END_OF_LINE) {
+    if (tokenlist[id].tick == token_ticks::END_OF_LINE) {
             compiler_data.error.add_line();
-        }
-        return back_token;
+    } 
+    if(id + 1 >= tokenlist.size()) {
+        unreach("token list out of range");
     }
+    return tokenlist[id++];
+}
+
+token token_lex::lexinteral() {
     if (*char_ptr == '#') {
         /*忽略注释*/
         while (!end_of_lex()) {
@@ -594,15 +601,23 @@ token_lex::token_lex(compiler_public_data& compiler_data)
 
 void token_lex::set_code(const std::string& code) {
     char_ptr = code.c_str();
-}
-
-token_lex::~token_lex() {
+    tokenlist.clear();
+    while(1) {
+        auto i = get_token();
+        tokenlist.push_back(i);
+        if(i.tick == token_ticks::END_OF_TOKENS) {
+            break;
+        }
+    }
     // 最后判断括号栈是否为空，如果不为空，说明括号未完全匹配，报错
     if (!check_brace.empty()) {
         char error_tmp[] = { check_brace.top(), '\0' };
         compiler_data.error.send_error_module(error::SyntaxError,
             language::error::syntaxerror_unmatched_char, error_tmp);
     }
+}
+
+token_lex::~token_lex() {
     compiler_data.error.reset_line();
 }
 }

src/TVM/types/trc_float.cpp

@@ -9,6 +9,7 @@ import TRE;
 import TVM.memory;
 import trc_int;
 import trc_string;
+import object;
 
 namespace trc::TVM_space::types {
 const RUN_TYPE_TICK trc_float::type = RUN_TYPE_TICK::float_T;

src/TVM/types/trc_int.cpp

@@ -8,6 +8,7 @@ import TVM.memory;
 import TVMdef;
 import TRE;
 import trcdef;
+import object;
 
 namespace trc::TVM_space::types {
 const RUN_TYPE_TICK trc_int::type = RUN_TYPE_TICK::int_T;

src/TVM/types/trc_long.cpp

@@ -12,6 +12,7 @@ import TVMdef;
 import trcdef;
 import TVM.memory;
 import unreach;
+import object;
 
 namespace trc::TVM_space::types {
 using namespace TVM_share;

src/TVM/types/trc_string.cpp

@@ -8,6 +8,7 @@ import TRE;
 import io;
 import TVM.memory;
 import trcdef;
+import object;
 
 namespace trc::TVM_space::types {
 using namespace TVM_share;