Skip to content

Commit

Permalink
fix(server): sample dataset were not correctly returned from datacata…
Browse files Browse the repository at this point in the history 
…logv3
  • Loading branch information
@rot1024
rot1024 committed Mar 25, 2024
1 parent a70309f commit 8dfb954
Show file tree
Hide file tree
Showing 13 changed files with 168 additions and 203 deletions.
22 changes: 12 additions & 10 deletions server/datacatalog/datacatalogv3/cms.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func (c *CMS) GetAll(ctx context.Context, project string) (*AllData, error) {
return c.GetGenericItems(ctx, project)
})

sampleItemsChan := lo.Async2(func() ([]*GenericItem, error) {
sampleItemsChan := lo.Async2(func() ([]*PlateauFeatureItem, error) {
return c.GetSampleItems(ctx, project)
})

Expand Down Expand Up @@ -104,7 +104,13 @@ func (c *CMS) GetAll(ctx context.Context, project string) (*AllData, error) {
if res := <-featureItemsChan; res.C != nil {
return nil, fmt.Errorf("failed to get feature items (%s): %w", res.A, res.C)
} else {
all.Plateau[res.A] = append(all.Plateau[res.A], res.B...)
for _, d := range res.B {
if d.Sample {
all.Sample = append(all.Sample, d)
} else {
all.Plateau[res.A] = append(all.Plateau[res.A], d)
}
}
}
}

Expand Down Expand Up @@ -147,7 +153,7 @@ func (c *CMS) GetPlateauItems(ctx context.Context, project, feature string) ([]*
items, err := getItemsAndConv(
c.cms, ctx, project, modelPrefix+feature,
func(i cms.Item) *PlateauFeatureItem {
return PlateauFeatureItemFrom(&i)
return PlateauFeatureItemFrom(&i, feature)
},
)
return items, err
Expand Down Expand Up @@ -180,18 +186,14 @@ func (c *CMS) GetGenericItems(ctx context.Context, project string) ([]*GenericIt
return items, err
}

func (c *CMS) GetSampleItems(ctx context.Context, project string) ([]*GenericItem, error) {
func (c *CMS) GetSampleItems(ctx context.Context, project string) ([]*PlateauFeatureItem, error) {
items, err := getItemsAndConv(
c.cms, ctx, project, modelPrefix+sampleModel,
func(i cms.Item) *GenericItem {
return GenericItemFrom(&i)
func(i cms.Item) *PlateauFeatureItem {
return PlateauFeatureItemFrom(&i, "")
},
)

for _, item := range items {
item.Category = "サンプルデータ"
}

return items, err
}

Expand Down
58 changes: 19 additions & 39 deletions server/datacatalog/datacatalogv3/cms_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"slices"
"strings"

"github.com/eukarya-inc/reearth-plateauview/server/cmsintegration/cmsintegrationcommon"
"github.com/eukarya-inc/reearth-plateauview/server/datacatalog/datacatalogcommon"
Expand Down Expand Up @@ -147,15 +146,16 @@ func (i *CityItem) IsPublicOrBeta() bool {
}

type PlateauFeatureItem struct {
ID string `json:"id,omitempty" cms:"id"`
City string `json:"city,omitempty" cms:"city,reference"`
CityGML string `json:"citygml,omitempty" cms:"citygml,-"`
Data []string `json:"data,omitempty" cms:"data,-"`
Desc string `json:"desc,omitempty" cms:"desc,textarea"`
Items []PlateauFeatureItemDatum `json:"items,omitempty" cms:"items,group"`
Dic string `json:"dic,omitempty" cms:"dic,textarea"`
Group string `json:"group,omitempty" cms:"group,text"`
MaxLOD string `json:"maxlod,omitempty" cms:"maxlod,-"`
ID string `json:"id,omitempty" cms:"id"`
City string `json:"city,omitempty" cms:"city,reference"`
CityGML string `json:"citygml,omitempty" cms:"citygml,-"`
Data []string `json:"data,omitempty" cms:"data,-"`
Desc string `json:"desc,omitempty" cms:"desc,textarea"`
Items []PlateauFeatureItemDatum `json:"items,omitempty" cms:"items,group"`
Dic string `json:"dic,omitempty" cms:"dic,textarea"`
Group string `json:"group,omitempty" cms:"group,text"`
MaxLOD string `json:"maxlod,omitempty" cms:"maxlod,-"`
FeatureType string `json:"feature_type,omitempty" cms:"feature_type,select"`
// metadata
Sample bool `json:"sample,omitempty" cms:"sample,bool,metadata"`
Status *cms.Tag `json:"status,omitempty" cms:"status,select,metadata"`
Expand Down Expand Up @@ -241,7 +241,7 @@ type DicEntry struct {
Order *int `json:"order"`
}

func PlateauFeatureItemFrom(item *cms.Item) (i *PlateauFeatureItem) {
func PlateauFeatureItemFrom(item *cms.Item, code string) (i *PlateauFeatureItem) {
i = &PlateauFeatureItem{}
item.Unmarshal(i)

Expand All @@ -251,6 +251,14 @@ func PlateauFeatureItemFrom(item *cms.Item) (i *PlateauFeatureItem) {
for ind, d := range i.Items {
i.Items[ind].Data = valueToAssetURLs(item.FieldByKeyAndGroup("data", d.ID).GetValue())
}
if i.FeatureType != "" {
// e.g. "建築物モデル（bldg）" -> Name="建築物モデル", FeatureType="bldg"
if _, ft := getLastBracketContent(i.FeatureType); ft != "" {
i.FeatureType = ft
}
} else {
i.FeatureType = code
}

return
}
Expand Down Expand Up @@ -298,10 +306,6 @@ type GenericItem struct {
Items []GenericItemDataset `json:"items,omitempty" cms:"items,group"`
OpenDataURL string `json:"open_data_url,omitempty" cms:"open_data_url,url"`
Category string `json:"category,omitempty" cms:"category,select"`
// sample
CityGML string `json:"citygml,omitempty" cms:"citygml,asset"`
FeatureType string `json:"feature_type,omitempty" cms:"-"`
MaxLODURL string `json:"maxlodUrl,omitempty" cms:"-"`
// metadata
Status *cms.Tag `json:"status,omitempty" cms:"status,select,metadata"`
Public bool `json:"public,omitempty" cms:"public,bool,metadata"`
Expand All @@ -317,16 +321,6 @@ func (c *GenericItem) Stage() stage {
return stageAlpha
}

func (c *GenericItem) IsPublic() bool {
stage := c.Stage()
return stage == stageGA
}

func (c *GenericItem) IsPublicOrBeta() bool {
stage := c.Stage()
return stage == stageGA || stage == stageBeta
}

type GenericItemDataset struct {
ID string `json:"id,omitempty" cms:"id"`
Name string `json:"name,omitempty" cms:"item_name,text"`
Expand All @@ -344,20 +338,6 @@ func GenericItemFrom(item *cms.Item) (i *GenericItem) {
for ind, d := range i.Items {
i.Items[ind].Data = valueToAssetURL(item.FieldByKeyAndGroup("data", d.ID).GetValue())
}

i.MaxLODURL = valueToAssetURL(item.FieldByKey("maxlod").GetValue())

// e.g. "建築物モデル（bldg）" -> Name="建築物モデル", FeatureType="bldg"
ft := lo.FromPtr(item.FieldByKey("feature_type").GetValue().String())
if strings.Contains(ft, "（") && strings.Contains(ft, "）") {
name, s, _ := strings.Cut(ft, "（")
s, _, _ = strings.Cut(s, "）")
i.FeatureType = s
if i.Name == "" {
i.Name = name
}
}

return
}

Expand Down
15 changes: 10 additions & 5 deletions server/datacatalog/datacatalogv3/cms_model_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,19 @@ func TestPlateauFeatureItemFrom(t *testing.T) {
Group: "item2",
Value: []any{"url4"}, // string is ignored
},
{
Key: "feature_type",
Value: "交通（道路）モデル（tran）",
},
},
}

expected := &PlateauFeatureItem{
ID: "id",
Data: []string{"url1", "url2"},
CityGML: "url_citygml",
MaxLOD: "url_maxlod",
ID: "id",
Data: []string{"url1", "url2"},
CityGML: "url_citygml",
MaxLOD: "url_maxlod",
FeatureType: "tran",
Items: []PlateauFeatureItemDatum{
{
ID: "item1",
Expand All @@ -61,7 +66,7 @@ func TestPlateauFeatureItemFrom(t *testing.T) {
},
}

assert.Equal(t, expected, PlateauFeatureItemFrom(item))
assert.Equal(t, expected, PlateauFeatureItemFrom(item, "code"))
}

func TestGenericItemFrom(t *testing.T) {
Expand Down
75 changes: 56 additions & 19 deletions server/datacatalog/datacatalogv3/conv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,33 +58,53 @@ func (all *AllData) Into() (res *plateauapi.InMemoryRepoContext, warning []strin
}

// plateau
plateauDatasetTypes := res.DatasetTypes.CodeMap(plateauapi.DatasetTypeCategoryPlateau)
plateauFeatureTypes := all.FeatureTypes.PlateauMap()
for _, dt := range res.DatasetTypes[plateauapi.DatasetTypeCategoryPlateau] {
ft := all.FeatureTypes.FindPlateauByCode(dt.GetCode())
datasets, w := convertPlateau(all.Plateau[dt.GetCode()], res.PlateauSpecs, dt, ft, ic)
datasets, w := convertPlateau(
all.Plateau[dt.GetCode()],
dt.GetCode(),
res.PlateauSpecs,
plateauDatasetTypes,
plateauFeatureTypes,
ic,
)
warning = append(warning, w...)
res.Datasets.Append(plateauapi.DatasetTypeCategoryPlateau, datasets)
}

// sample
sample := res.DatasetTypes.FindByCode("sample", plateauapi.DatasetTypeCategoryGeneric).(*plateauapi.GenericDatasetType)
if sample != nil {
datasets, w := convertPlateauRaw(
all.Sample,
"",
res.PlateauSpecs,
plateauDatasetTypes,
plateauFeatureTypes,
ic,
)
warning = append(warning, w...)
res.Datasets.Append(
plateauapi.DatasetTypeCategoryGeneric,
plateauapi.ToDatasets(plateauapi.PlateauDatasetsToGenericDatasets(datasets, sample.ID, sample.Code, "sample")),
)
}

// related
{
datasets, w := convertRelated(all.Related, res.DatasetTypes[plateauapi.DatasetTypeCategoryRelated], ic)
warning = append(warning, w...)
res.Datasets.Append(plateauapi.DatasetTypeCategoryRelated, datasets)
}

// generic and sample
// generic
{
src := append(all.Generic, all.Sample...)
datasets, w := convertGeneric(src, res.DatasetTypes[plateauapi.DatasetTypeCategoryGeneric], ic)
datasets, w := convertGeneric(all.Generic, res.DatasetTypes[plateauapi.DatasetTypeCategoryGeneric], ic)
warning = append(warning, w...)
res.Datasets.Append(plateauapi.DatasetTypeCategoryGeneric, datasets)
}

// move plateau sample data to generic
sampleType, _ := res.DatasetTypes.FindByCode(
"sample", plateauapi.DatasetTypeCategoryGeneric).(*plateauapi.GenericDatasetType)
movePlateauSampleDataToGeneric(res.Datasets, sampleType)

// citygml
{
var w []string
Expand All @@ -110,16 +130,33 @@ func getWards(items []*PlateauFeatureItem, ic *internalContext) (res []*plateaua
return
}

func convertPlateau(items []*PlateauFeatureItem, specs []plateauapi.PlateauSpec, dt plateauapi.DatasetType, ft *FeatureType, ic *internalContext) (res []plateauapi.Dataset, warning []string) {
pdt, ok := dt.(*plateauapi.PlateauDatasetType)
if !ok {
warning = append(warning, fmt.Sprintf("plateau %s: invalid dataset type: %s", dt.GetCode(), dt.GetName()))
return
}

layerNames := ic.layerNamesForType[pdt.Code]
func convertPlateau(items []*PlateauFeatureItem, code string, specs []plateauapi.PlateauSpec, dts map[string]plateauapi.DatasetType, fts map[string]*FeatureType, ic *internalContext) ([]plateauapi.Dataset, []string) {
res, w := convertPlateauRaw(items, code, specs, dts, fts, ic)
return plateauapi.ToDatasets(res), w
}

func convertPlateauRaw(items []*PlateauFeatureItem, code string, specs []plateauapi.PlateauSpec, dts map[string]plateauapi.DatasetType, fts map[string]*FeatureType, ic *internalContext) (res []*plateauapi.PlateauDataset, warning []string) {
for _, ds := range items {
code := code
if ds.FeatureType != "" {
code = ds.FeatureType
}

dt := dts[code]
ft := fts[code]

if dt == nil || ft == nil {
warning = append(warning, fmt.Sprintf("plateau %s: invalid feature type: %s", ds.ID, code))
continue
}

pdt, ok := dt.(*plateauapi.PlateauDatasetType)
if !ok {
warning = append(warning, fmt.Sprintf("plateau %s: invalid dataset type: %s", dt.GetCode(), dt.GetName()))
return
}

layerNames := ic.layerNamesForType[pdt.Code]
area := ic.AreaContext(ds.City)
if area == nil {
warning = append(warning, fmt.Sprintf("plateau %s %s: invalid city: %s", ds.ID, pdt.Code, ds.City))
Expand All @@ -142,7 +179,7 @@ func convertPlateau(items []*PlateauFeatureItem, specs []plateauapi.PlateauSpec,
FeatureType: ft,
Year: ic.regYear,
}
ds, w := ds.toDatasets(opts)
ds, w := ds.toDatasetsRaw(opts)
warning = append(warning, w...)
if ds != nil {
res = append(res, ds...)
Expand Down
9 changes: 2 additions & 7 deletions server/datacatalog/datacatalogv3/conv_citygml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func toCityGMLs(all *AllData, regYear int) (map[plateauapi.ID]*plateauapi.CityGM
}

for _, d := range all.Sample {
if d.MaxLODURL == "" || d.CityGML == "" {
if d.MaxLOD == "" || d.CityGML == "" {
continue
}

Expand All @@ -87,12 +87,7 @@ func toCityGMLs(all *AllData, regYear int) (map[plateauapi.ID]*plateauapi.CityGM
continue
}

stage, _ := citygml.Admin.(map[string]any)["stage"].(string)
if stage == string(stageGA) && !d.IsPublic() || stage == string(stageBeta) && !d.IsPublicOrBeta() {
continue
}

addCityGML(d.CityGML, d.MaxLODURL, d.FeatureType, citygml)
addCityGML(d.CityGML, d.MaxLOD, d.FeatureType, citygml)
}

return res, nil
Expand Down
46 changes: 0 additions & 46 deletions server/datacatalog/datacatalogv3/conv_dataset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,49 +225,3 @@ func assetRootPath(p string) string {
fn := strings.TrimSuffix(path.Base(p), path.Ext(p))
return path.Join(path.Dir(p), fn)
}

func sampleAdmin(sample bool) any {
if sample {
return map[string]any{"sample": true}
}
return nil
}

func isSampleAdmin(admin any) bool {
m, _ := admin.(map[string]any)
if m == nil {
return false
}

_, ok := m["sample"]
return ok
}

func movePlateauSampleDataToGeneric(d plateauapi.Datasets, t *plateauapi.GenericDatasetType) {
if t == nil {
return
}

plateau := d[plateauapi.DatasetTypeCategoryPlateau]
generic := d[plateauapi.DatasetTypeCategoryGeneric]

for i, p := range plateau {
pd, ok := p.(*plateauapi.PlateauDataset)
if !ok || !isSampleAdmin(p.GetAdmin()) {
continue
}

newID := pd.ID + "_sample"
tid := t.ID
tcode := t.Code
gd := plateauapi.PlateauDatasetToGenericDataset(pd, tid, tcode, newID)
gd.Admin.(map[string]any)["plateauTypeId"] = pd.TypeID
gd.Admin.(map[string]any)["plateauTypeCode"] = pd.TypeCode

generic = append(generic, gd)
plateau = append(plateau[:i], plateau[i+1:]...)
}

d[plateauapi.DatasetTypeCategoryPlateau] = plateau
d[plateauapi.DatasetTypeCategoryGeneric] = generic
}
7 changes: 6 additions & 1 deletion server/datacatalog/datacatalogv3/conv_dataset_plateau.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,12 @@ type ToPlateauDatasetsOptions struct {
Year int
}

func (i *PlateauFeatureItem) toDatasets(opts ToPlateauDatasetsOptions) (res []plateauapi.Dataset, warning []string) {
func (i *PlateauFeatureItem) toDatasets(opts ToPlateauDatasetsOptions) ([]plateauapi.Dataset, []string) {
res, w := i.toDatasetsRaw(opts)
return plateauapi.ToDatasets(res), w
}

func (i *PlateauFeatureItem) toDatasetsRaw(opts ToPlateauDatasetsOptions) (res []*plateauapi.PlateauDataset, warning []string) {
if !opts.Area.IsValid() {
warning = append(warning, fmt.Sprintf("plateau %s: invalid area", i.ID))
return
Expand Down
Loading

0 comments on commit 8dfb954

Please sign in to comment.