diff --git a/server/datacatalog/datacatalogv3/cms_model.go b/server/datacatalog/datacatalogv3/cms_model.go index 891d8c3b1..a7b5d5db1 100644 --- a/server/datacatalog/datacatalogv3/cms_model.go +++ b/server/datacatalog/datacatalogv3/cms_model.go @@ -430,10 +430,11 @@ func geospatialjpURL(cityCode string, cityName string, year int) string { } type GeospatialjpDataItem struct { - ID string `json:"id,omitempty" cms:"id"` - City string `json:"city,omitempty" cms:"city,reference"` - CityGML string `json:"citygml,omitempty" cms:"citygml,asset"` - MaxLOD string `json:"maxlod,omitempty" cms:"maxlod,asset"` + ID string `json:"id,omitempty" cms:"id"` + City string `json:"city,omitempty" cms:"city,reference"` + CityGML string `json:"citygml,omitempty" cms:"citygml,asset"` + MaxLOD string `json:"maxlod,omitempty" cms:"maxlod,asset"` + HasIndex bool `json:"has_index,omitempty" cms:"-"` } func GeospatialjpDataItemFrom(item *cms.Item) *GeospatialjpDataItem { @@ -442,6 +443,7 @@ func GeospatialjpDataItemFrom(item *cms.Item) *GeospatialjpDataItem { City string `json:"city,omitempty" cms:"city,reference"` CityGML any `json:"citygml,omitempty" cms:"citygml,asset"` MaxLOD any `json:"maxlod,omitempty" cms:"maxlod,asset"` + Index string `json:"desc_index,omitempty" cms:"desc_index,markdown"` } it := itemType{} @@ -451,9 +453,10 @@ func GeospatialjpDataItemFrom(item *cms.Item) *GeospatialjpDataItem { maxlod := anyToAssetURL(it.MaxLOD) return &GeospatialjpDataItem{ - ID: it.ID, - City: it.City, - CityGML: citygml, - MaxLOD: maxlod, + ID: it.ID, + City: it.City, + CityGML: citygml, + MaxLOD: maxlod, + HasIndex: it.Index != "", } } diff --git a/server/datacatalog/datacatalogv3/cms_test.go b/server/datacatalog/datacatalogv3/cms_test.go new file mode 100644 index 000000000..a47cad8e5 --- /dev/null +++ b/server/datacatalog/datacatalogv3/cms_test.go @@ -0,0 +1,62 @@ +package datacatalogv3 + +import ( + "context" + "encoding/csv" + "os" + "testing" + + "github.com/joho/godotenv" + cms "github.com/reearth/reearth-cms-api/go" + "github.com/stretchr/testify/assert" +) + +func TestExtractDataFromCMS(t *testing.T) { + run := false + + if !run { + t.Skip("skip") + } + + _ = godotenv.Load("../../.env") + cmsurl := os.Getenv("REEARTH_PLATEAUVIEW_CMS_BASEURL") + cmstoken := os.Getenv("REEARTH_PLATEAUVIEW_CMS_TOKEN") + prj := os.Getenv("REEARTH_PLATEAUVIEW_TEST_CMS_PROJECT") + if cmsurl == "" || cmstoken == "" || prj == "" { + t.Skip("cms url or cms token or project is empty") + } + + ctx := context.Background() + c, err := cms.New(cmsurl, cmstoken) + assert.NoError(t, err) + + c2 := NewCMS(c, 2023) + all, err := c2.GetAll(ctx, prj) + assert.NoError(t, err) + + t.Log("get all data done") + + // do something with all + records := [][]string{} + + for _, city := range all.City { + g := all.FindGspatialjpDataItemByCityID(city.ID) + if g == nil { + continue + } + + if !g.HasIndex { + continue + } + + r := []string{city.ID, city.CityName, city.CityCode} + records = append(records, r) + } + + f, err := os.Create("cities.csv") + assert.NoError(t, err) + w := csv.NewWriter(f) + _ = w.WriteAll(records) + w.Flush() + _ = f.Close() +} diff --git a/server/datacatalog/datacatalogv3/model.go b/server/datacatalog/datacatalogv3/model.go index a075e4cc4..2fb00d63f 100644 --- a/server/datacatalog/datacatalogv3/model.go +++ b/server/datacatalog/datacatalogv3/model.go @@ -34,6 +34,15 @@ func (all *AllData) FeatureTypesOf(cityID string) (res []string) { return res } +func (d *AllData) FindGspatialjpDataItemByCityID(cityID string) *GeospatialjpDataItem { + for _, i := range d.GeospatialjpDataItems { + if i != nil && i.City == cityID { + return i + } + } + return nil +} + type FeatureTypes struct { Plateau []FeatureType Related []FeatureType diff --git a/worker/preparegspatialjp/cms_wrapper.go b/worker/preparegspatialjp/cms_wrapper.go index 7fa9f7ea4..8257d92e7 100644 --- a/worker/preparegspatialjp/cms_wrapper.go +++ b/worker/preparegspatialjp/cms_wrapper.go @@ -169,3 +169,7 @@ func (c *CMSWrapper) Comment(ctx context.Context, comment string) { log.Errorfc(ctx, "failed to comment to %s: %v", c.CityItemID, err) } } + +func (c *CMSWrapper) Commentf(ctx context.Context, f string, args ...any) { + c.Comment(ctx, fmt.Sprintf(f, args...)) +} diff --git a/worker/preparegspatialjp/command.go b/worker/preparegspatialjp/command.go index 455151295..1f5eeb39c 100644 --- a/worker/preparegspatialjp/command.go +++ b/worker/preparegspatialjp/command.go @@ -47,7 +47,7 @@ func CommandSingle(conf *Config) (err error) { return fmt.Errorf("failed to initialize CMS client: %w", err) } - // get items fron CNS + // get items fron CMS log.Infofc(ctx, "getting item from CMS...") cityItemRaw, err := cms.GetItem(ctx, conf.CityItemID, true) @@ -63,19 +63,6 @@ func CommandSingle(conf *Config) (err error) { return fmt.Errorf("invalid city item: %s", conf.CityItemID) } - if cityItem.YearInt() == 0 { - return fmt.Errorf("invalid year: %s", cityItem.Year) - } - - if cityItem.SpecVersionMajorInt() == 0 { - return fmt.Errorf("invalid spec version: %s", cityItem.Spec) - } - - uc := GetUpdateCount(cityItem.CodeLists) - if uc == 0 { - return fmt.Errorf("invalid update count: %s", cityItem.CodeLists) - } - indexItemRaw, err := cms.GetItem(ctx, cityItem.GeospatialjpIndex, false) if err != nil { return fmt.Errorf("failed to get index item: %w", err) @@ -120,6 +107,22 @@ func CommandSingle(conf *Config) (err error) { WetRun: conf.WetRun, } + if cityItem.YearInt() == 0 { + cw.Commentf(ctx, "公開準備処理を開始できません。整備年度が不正です: %s", cityItem.Year) + return fmt.Errorf("invalid year: %s", cityItem.Year) + } + + if cityItem.SpecVersionMajorInt() == 0 { + cw.Commentf(ctx, "公開準備処理を開始できません。仕様書バージョンが不正です: %s", cityItem.Spec) + return fmt.Errorf("invalid spec version: %s", cityItem.Spec) + } + + uc := GetUpdateCount(cityItem.CodeLists) + if uc == 0 { + cw.Commentf(ctx, "公開準備処理を開始できません。codeListsのzipファイルの命名規則が不正のため版数を読み取れませんでした。もう一度ファイル名の命名規則を確認してください。_1_op_のような文字が必須です。: %s", cityItem.CodeLists) + return fmt.Errorf("invalid update count: %s", cityItem.CodeLists) + } + tmpDirName := fmt.Sprintf("%s-%d", time.Now().Format("20060102-150405"), rand.Intn(1000)) tmpDir := filepath.Join(tmpDirBase, tmpDirName) log.Infofc(ctx, "tmp dir: %s", tmpDir)