Merge pull request #179 from psrenergy/jg/parser_merge

Fix MERGE_CLASS

src/PMD/PMD.jl

@@ -28,8 +28,10 @@ const DataStruct = Dict{String, Dict{String, Attribute}}
 include("model_template.jl")
 include("relation_mapper.jl")
 
+# TODO non portable
 const _PMDS_BASE_PATH = joinpath(@__DIR__(), "pmds")
 
+# TODO non portable
 const PMD_MODEL_TEMPLATES_PATH =
     joinpath(@__DIR__(), "..", "json_metadata", "modeltemplates.sddp.json")
 

src/PMD/model_template.jl

@@ -1,5 +1,19 @@
 using JSON
 
+
+"""
+    ModelTemplate
+
+struct ModelTemplate
+    map::Dict{String, Set{String}}
+    inv::Dict{String, String}
+end
+
+Data structure to store the model template information.
+The keys of `map` contain the class name, for instance, `PSRHydroPlant`, the values
+contain the models names in the pmd files.
+
+"""
 struct ModelTemplate
     map::Dict{String, Set{String}}
     inv::Dict{String, String}
@@ -27,6 +41,7 @@ Base.iterate(mt::ModelTemplate, i::Integer) = iterate(mt.map, i)
 _hasmap(mt::ModelTemplate, k::AbstractString) = haskey(mt.map, string(k))
 _hasinv(mt::ModelTemplate, v::AbstractString) = haskey(mt.inv, string(v))
 
+# TODO - document file output format
 function dump_model_template(path::String, model_template::ModelTemplate)
     list = []
 
@@ -55,6 +70,8 @@ function load_model_template(path::AbstractString)
     return model_template
 end
 
+# TODO
+# in-place operations have the first argument as the data that will be modified.
 function load_model_template!(path::AbstractString, model_template::ModelTemplate)
     raw_struct = JSON.parsefile(path)
 

src/PMD/parser/parser.jl

@@ -7,7 +7,7 @@ mutable struct Parser
     # original pmd path and current line number, used to
     # compose better error/warning messages
     path::String
-    lineno::Int
+    line_number::Int
 
     # current parser state
     state::Vector{Any}
@@ -19,8 +19,12 @@ mutable struct Parser
     verbose::Bool
 
     # Study info
+    # data structure containgin the class and attribute definitions to be filled
+    # by the parser
     data_struct::DataStruct
+    # contains the relations between the classes obtaned from the json file OR from pmd
     relation_mapper::RelationMapper
+    # maps classes like PSRHydroPlant to especific models like MODL:SDDP_V10.2_Bateria
     model_template::ModelTemplate
 
     function Parser(
@@ -34,7 +38,7 @@ mutable struct Parser
         return new(
             io,
             path,
-            0,                          # lineno
+            0,                          # line_number
             [],                         # state
             Dict{String, Vector{Any}}(), # merge
             verbose,
@@ -71,22 +75,22 @@ function _syntax_error(
     parser::Parser,
     msg::AbstractString,
 )
-    return error("Syntax Error in '$(parser.path):$(parser.lineno)': $msg")
+    return error("Syntax Error in '$(parser.path):$(parser.line_number)': $msg")
 end
 
 function _error(
     parser::Parser,
     msg::AbstractString,
 )
-    return error("Error in '$(parser.path):$(parser.lineno)': $msg")
+    return error("Error in '$(parser.path):$(parser.line_number)': $msg")
 end
 
 function _warning(
     parser::Parser,
     msg::AbstractString,
 )
     if parser.verbose
-        @warn("In '$(parser.path):$(parser.lineno)': $msg")
+        @warn("In '$(parser.path):$(parser.line_number)': $msg")
     end
 end
 
@@ -192,9 +196,9 @@ end
         verbose::Bool = false,
     )
 
+It is intended to be useful when **writing tests** for the parser.
 Parses a PMD file with as little context as possible.
 Besides the file path, only the model template is necessary.
-It is intended to be useful when writing tests for the parser.
 """
 function parse end
 
@@ -231,7 +235,7 @@ end
 
 function parse!(parser::Parser)
     for line in readlines(parser.io)
-        parser.lineno += 1
+        parser.line_number += 1
 
         # remove comments & trailing spaces
         m = match(r"^\s*(.*?)\s*(\/\/.*)?$", line)
@@ -243,8 +247,9 @@ function parse!(parser::Parser)
         _parse_line!(parser, m[1], _get_state(parser))
     end
 
+    # after parsing, we should be in the idle state
     if !(_get_state(parser) isa PMD_IDLE)
-        _syntax_error(parser, "Unexpected EOF")
+        _syntax_error(parser, "Unexpected parsing completion")
     end
 
     # apply merges
@@ -262,15 +267,23 @@ function _parse_line!(parser::Parser, line::AbstractString, ::PMD_IDLE)
     if m !== nothing
         model_name = String(m[2])
 
+        # obtain the collection (class) linked to a given model in this database
         collection = get(parser.model_template.inv, model_name, model_name)
 
+        _warning(parser, "DEFINE_MODEL '$model_name' for class '$collection'")
+
+        # TODO - verify if already existed? and throw error?
+        if haskey(parser.data_struct, collection)
+            _warning(parser, "Replacing definition of class '$collection' fwith model '$model_name'")
+        end
         parser.data_struct[collection] = Dict{String, Attribute}()
 
         # default attributes that belong to "all collections"
         parser.data_struct[collection]["name"] = Attribute("name", false, String, 0, "")
         parser.data_struct[collection]["code"] = Attribute("code", false, Int32, 0, "")
         parser.data_struct[collection]["AVId"] = Attribute("AVId", false, String, 0, "")
 
+        # special attributes from specific classes
         if collection == "PSRSystem"
             parser.data_struct[collection]["id"] = Attribute("id", false, String, 0, "")
         end
@@ -286,6 +299,12 @@ function _parse_line!(parser::Parser, line::AbstractString, ::PMD_IDLE)
     if m !== nothing
         collection = String(m[1])
 
+        _warning(parser, "DEFINE_CLASS '$collection'")
+
+        # TODO - verify if already existed? and throw error?
+        if haskey(parser.data_struct, collection)
+            _warning(parser, "Replacing definition of class '$collection'")
+        end
         parser.data_struct[collection] = Dict{String, Attribute}()
 
         # default attributes that belong to "all collections"
@@ -309,21 +328,15 @@ function _parse_line!(parser::Parser, line::AbstractString, ::PMD_IDLE)
         collection = String(m[1])
         model_name = String(m[2])
 
-        if _hasinv(parser.model_template, model_name)
-            _cache_merge!(parser, collection, model_name)
-
-            _push_state!(parser, PMD_MERGE_CLASS(collection))
+        _warning(parser, "MERGE_CLASS to add new attributes from temporary '$model_name' to existing class '$collection'")
 
-            return nothing
-        else
-            _warning(parser, "Unknown model '$(model_name)'")
+        if !haskey(parser.data_struct, collection)
+            _error(parser, "Class '$collection no found. Consider changing pmd load order.'")
+        end
 
-            # By setting the collection to 'nothing', we are telling
-            # the parser to ignore the block and its contents
-            _push_state!(parser, PMD_MERGE_CLASS(nothing))
+        _push_state!(parser, PMD_MERGE_CLASS(collection))
 
-            return nothing
-        end
+        return nothing
     end
 
     return _syntax_error(parser, "Invalid input: '$line'")

src/PMD/parser/states.jl

@@ -17,6 +17,18 @@ struct PMD_IDLE <: ParserState end
     PMD_DEF_MODEL
 
 Indicates that the parser is parsing a _model definition_ block.
+Identified by `DEFINE_MODEL XXX` for a model named XXX.
+
+Example
+
+```
+DEFINE_MODEL MODL:SDDP_V10.2_Bateria
+	PARM REAL 	Einic
+END_MODEL
+```
+
+The model name is `MODL:SDDP_V10.2_Bateria`.
+The model can be attached to a class to define its attributes.
 """
 struct PMD_DEF_MODEL <: ParserState
     collection::Union{String, Nothing}
@@ -26,6 +38,17 @@ end
     PMD_DEF_CLASS
 
 Indicates that the parser is parsing a _class definition_ block.
+
+Example
+
+```
+DEFINE_CLASS Contract_Forward
+  PARM INTEGER SpreadUnit
+END_CLASS
+```
+
+The class name is `Contract_Forward`.
+This class definition does not require a model attached.
 """
 struct PMD_DEF_CLASS <: ParserState
     collection::Union{String, Nothing}
@@ -43,13 +66,15 @@ end
 """
     PMD_DEF_VALIDATION
 
-Validation block.
+Validation block `DEFINE_VALIDATION`.
+This block is skipped.
 """
 struct PMD_DEF_VALIDATION <: ParserState end
 
 """
     PMD_DEF_MATH
 
-Math block.
+Math block `DEFINE_MATH`.
+This is skipped.
 """
 struct PMD_DEF_MATH <: ParserState end

src/PMD/relation_mapper.jl

@@ -1,5 +1,6 @@
 using JSON
 
+# TODO non portable
 const _DEFAULT_RELATIONS_PATH =
     joinpath(@__DIR__(), "..", "json_metadata", "relations.default.json")
 
@@ -50,9 +51,9 @@ const RELATION_TABLE = Dict{String, RelationType}(
     Relation(type::RelationType, attribute::String)
 """
 struct Relation
-    type::RelationType # deprecated ?
-    attribute::String
-    is_vector::Bool
+    type::RelationType # deprecated ? # attribute type originally defined in PSRClasses
+    attribute::String # relation name
+    is_vector::Bool # relation maps the current element to a vector of elements or to a single element'
 
     function Relation(type::RelationType, attribute::String)
         return new(type, attribute, type == RELATION_1_TO_N || type == RELATION_BACKED)

src/json_metadata/modeltemplates.sddp.json

@@ -220,5 +220,11 @@
         "models": [
             "SDDP_V10.2_RestricaoSoma"
         ]
+    },
+    {
+        "classname": "PSRExpansionProject",
+        "models": [
+            "Optgen_ProjetoExpansao"
+        ]
     }
 ]

test/data/pmd/source1.pmd

@@ -1,6 +1,10 @@
 // ---------------------
 // --- Some comments ---
 // ---------------------
+DEFINE_CLASS PSRHydroPlant
+	PARM   INTEGER   test
+END_CLASS
+
 DEFINE_CLASS PSRGeneratorUnit
 	PARM   INTEGER   NumberUnits
 	PARM   INTEGER   Code        @id
@@ -24,7 +28,6 @@ MERGE_CLASS PSRHydroPlant Optfolio_HydroPlant
 	VECTOR  REFERENCE  Costs         Optfolio_GenericCosts
 	PARM    INTEGER Included
 	PARM	REAL	Percentage
-	PARM	INTEGER flagMRE
 END_CLASS
 
 // Comment

test/model_template.jl

@@ -45,6 +45,7 @@ function test_model_template1()
 
     mt_copy_path = joinpath(temp_path, "modeltemplate.copy.json")
 
+    # TODO - test resulting file at `mt_copy_path`
     PSRI.PMD.dump_model_template(mt_copy_path, data)
 
     data_copy = PSRI.load_study(

test/pmd_parser.jl

@@ -210,6 +210,26 @@ function test_pmd_source_1()
         @test haskey(data["PSRHydroPlant"], "Included")
 
         @test data == Dict{String, Dict{String, PSRI.PMD.Attribute}}(
+            "PSRHydroPlant" => Dict(
+                "AVId" =>
+                    PSRI.PMD.Attribute("AVId", false, String, 0, ""),
+                "name" =>
+                    PSRI.PMD.Attribute("name", false, String, 0, ""),
+                "code" =>
+                    PSRI.PMD.Attribute("code", false, Int32, 0, ""),
+                "test" =>
+                    PSRI.PMD.Attribute("test", false, Int32, 0, ""),
+                "DataSensib" =>
+                    PSRI.PMD.Attribute("DataSensib", true, Dates.Date, 0, ""),
+                "MinCOD" =>
+                    PSRI.PMD.Attribute("MinCOD", false, Dates.Date, 0, ""),
+                "SensibPotInst" =>
+                    PSRI.PMD.Attribute("SensibPotInst", true, Float64, 0, "DataSensib"),
+                "Included" =>
+                    PSRI.PMD.Attribute("Included", false, Int32, 0, ""),
+                "Percentage" =>
+                    PSRI.PMD.Attribute("Percentage", false, Float64, 0, ""),
+            ),
             "Contract_Forward" => Dict(
                 "AVId" =>
                     PSRI.PMD.Attribute("AVId", false, String, 0, ""),