wmo-im · tomkralidis · Nov 20, 2023 · Nov 14, 2023 · Nov 15, 2023 · Nov 15, 2023
diff --git a/.github/workflows/bundle.yml b/.github/workflows/bundle.yml
@@ -6,6 +6,9 @@ on:
     - main
     paths:
     - 'topic-hierarchy/**.csv'
+  pull_request:
+    branches:
+    - main
 
 jobs:
   main:
@@ -19,20 +22,17 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install requirements 📦
       run: |
-        python3 -m pip install --upgrade pip
-        pip3 install https://github.com/wmo-im/pywcmp/archive/master.zip
-        pip3 install -U jsonschema
-    - name: Cache resources bundle 📦
-      run: pywcmp bundle sync
+    - name: Create resources bundle 📦
+      run: |
+        python3 scripts/generate-bundle.py
+        zip -j /tmp/wth-bundle.zip topic-hierarchy/*.csv
     - uses: actions/checkout@v3
       with:
         ref: gh-pages
     - name: update gh-pages branch and publish
-      run: | 
+      run: |
+        mv /tmp/wth-bundle.zip .
         git config --global user.email "[email protected]"
         git config --global user.name "Tom Kralidis"
-        zip -j /tmp/all.json.zip ~/.pywcmp/wis2-topic-hierarchy/all.json
-        mv /tmp/all.json.zip .
-        ls -alh
-        git commit -m "generate bundle" all.json.zip
-        git push -f origin main
+        git commit -m "generate bundle" wth-bundle.zip
+        git push -f origin gh-pages
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 *.html
 *.pdf
 .DS_Store
+topic-hierarchy/earth-system-discipline.csv
diff --git a/scripts/generate-bundle.py b/scripts/generate-bundle.py
@@ -0,0 +1,50 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+import csv
+import os
+
+topics = []
+
+for root, dirs, files in os.walk('topic-hierarchy/earth-system-discipline'):
+    for file_ in files:
+        if file_.endswith('.csv'):
+            root2 = root.replace('topic-hierarchy/earth-system-discipline', '')
+            if root2.startswith('/'):
+                root2 = root2.replace('/', '', 1)
+            filename = os.path.join(root, file_)
+            with open(filename) as fh2:
+                reader = csv.reader(fh2)
+                next(reader)
+                for row in reader:
+                    topic_to_add = f'{root2}/{row[0]}'
+                    if topic_to_add.startswith('/'):
+                        topic_to_add = topic_to_add.lstrip('/')
+
+                    topics.append(topic_to_add)
+
+with open('topic-hierarchy/earth-system-discipline.csv', 'w') as fh:
+    fieldnames = ['Name']
+    writer = csv.DictWriter(fh, fieldnames=fieldnames)
+    writer.writeheader()
+
+    for topic in sorted(topics):
+        writer.writerow({'Name': topic})
diff --git a/standard/recommendations/core/PER_publishing.adoc b/standard/recommendations/core/PER_publishing.adoc
@@ -3,4 +3,5 @@
 |===
 ^|*Permission {counter:req-id}* |*/per/core/publishing*
 ^|A |Metadata MAY be published at any level at or below the notification type (``metadata``).
+^|B |Experimental data MAY be published at the ``experimental`` level of a given domain, for domain topics which are not yet approved.
 |===
diff --git a/standard/recommendations/core/REC_publishing.adoc b/standard/recommendations/core/REC_publishing.adoc
@@ -0,0 +1,6 @@
+[[rec_core_publishing]]
+[width="90%",cols="2,6a"]
+|===
+^|*Recommendation {counter:req-id}* |*/rec/core/publishing*
+^|A |The domain ``experimental`` level SHOULD be used as a temporary approach until a given domain topic is approved.
+|===
diff --git a/standard/requirements/core/REQ_management.adoc b/standard/requirements/core/REQ_management.adoc
@@ -4,8 +4,9 @@
 ^|*Requirement {counter:req-id}* |*/req/core/management*
 ^|A |Primary levels 1-8 SHALL be determined by the WMO Task Team on WIS Metadata.
 ^|B |Domain specific levels (level 8 and beyond) SHALL be determined by domain experts and user communities.
-^|C |Domain specific levels (level 8 and beyond) SHALL be coordinated and integrated by the WMO Task Team on WIS Metadata.
-^|D
+^|C |Domain specific levels (level 8 and beyond) SHALL be defined using a hierarchical approach.
+^|D |Domain specific levels (level 8 and beyond) SHALL be coordinated and integrated by the WMO Task Team on WIS Metadata.
+^|E
 a|Domain specific level updates SHALL be implemented using the following steps:
 
 - domain specific teams submit proposals to TT-WISMD

diff --git a/standard/sections/clause_7_normative_text.adoc b/standard/sections/clause_7_normative_text.adoc
@@ -18,8 +18,8 @@ The WTH is managed by the WMO Task Team on WIS Metadata. Updates to the WTH are
 
 The WIS Topic Hierarchy has been developed according to the classification of the Earth System domains in Annex 1 of Resolution 1 Cg-Ext(2021), and it is managed in two parts:
 
-1. Primary topic levels (levels 1-7): topic structure applies to all data and services in WIS
-2. Domain specific topic subcategory levels (level 8 and beyond): topic structure proposed by domain experts and user communities. Note that the number of levels in this part may vary according to the requirements of various domains.
+1. Primary topic levels (levels 1-7): topic structure applies to all data and services in WIS.  They are relational, meaning that any combination of each level can be used to construct a topic applicable to a data or metadata notification
+2. Domain specific topic subcategory levels (level 8 and beyond): topic structures proposed by domain experts and user communities.  Note that the number of levels in this part may vary according to the requirements of various domains, and are meant to represent a hierarchical representation of data publication from a given domain.
 
 include::../requirements/core/REQ_management.adoc[]
 
@@ -45,7 +45,8 @@ include::../recommendations/core/PER_centre-id.adoc[]
 
 ==== Publishing guidelines
 
-For maximum utility and efficient management of topics, data, metadata, and reports need to be published to a minimum level.  This helps with avoiding "pollution" of messages in higher level topics which are used to delineate core concepts (centre identification, notification types, data policy, etc.).
+For maximum utility and efficient management of topics, data, metadata, and reports need to be published to a minimum level.  This helps with avoiding "pollution" of messages in higher level topics which are used to delineate core concepts (centre identification, notification types, data policy, etc.).  Note that each domain is designated a topic named ``experimental`` for publication to provisional topics.
 
 include::../requirements/core/REQ_publishing.adoc[]
+include::../recommendations/core/REC_publishing.adoc[]
 include::../recommendations/core/PER_publishing.adoc[]
diff --git a/topic-hierarchy.csv b/topic-hierarchy.csv
@@ -6,4 +6,3 @@ Level,Name,Description
 5,notification-type,"WIS2 notification types (data, metadata, reports [from monitoring activities])"
 6,data-policy,Data policy as defined by the WMO Unified Data Policy. core data are available from the Global Caches with open access on a free and unrestricted basis. Notifications for core and recommended data are available by subscription to Global Brokers. recommended data are downloaded from the original NC/DCPC and may require authentication/authorisation
 7,earth-system-discipline,As per Annex 1 of resolution 1 Cg-Ext-2021
-8,earth-system-discipline-subcategory,As proposed by domain experts and further approved by INFCOM
diff --git a/topic-hierarchy/atmospheric-composition.csv b/topic-hierarchy/atmospheric-composition.csv
diff --git a/topic-hierarchy/centre-id.csv b/topic-hierarchy/centre-id.csv
@@ -1,7 +1,7 @@
-Name,Description,Child
-ca-eccc-msc,"Environment and Climate Change Canada, Meteorological Service of Canada",notification-type
-cu-insmet,Institute of Meteorology of Cuba,notification-type
-de-dwd,Deutscher Wetterdienst,notification-type
-fr-meteo-france,Météo-France,notification-type
-int-ecmwf,European Centre for Medium-Range Weather Forecasts,notification-type
-org-woudc,World Ozone and Ultraviolet Radiation Data Centre,notification-type
+Name,Description
+ca-eccc-msc,"iEnvironment and Climate Change Canada, Meteorological Service of Canada"
+cu-insmet,Institute of Meteorology of Cuba
+de-dwd,Deutscher Wetterdienst
+fr-meteo-france,Météo-France
+int-ecmwf,European Centre for Medium-Range Weather Forecasts
+org-woudc,World Ozone and Ultraviolet Radiation Data Centre
diff --git a/topic-hierarchy/channel.csv b/topic-hierarchy/channel.csv
@@ -1,3 +1,3 @@
-Name,Description,Child
-origin,data downloaded from original NC/DCPC,version
-cache,data downloaded from a Global Cache,version
+Name,Description
+origin,data downloaded from original NC/DCPC
+cache,data downloaded from a Global Cache
diff --git a/topic-hierarchy/climate.csv b/topic-hierarchy/climate.csv
diff --git a/topic-hierarchy/cryosphere.csv b/topic-hierarchy/cryosphere.csv
diff --git a/topic-hierarchy/data-policy.csv b/topic-hierarchy/data-policy.csv
@@ -1,3 +1,3 @@
-Name,Description,Child
-core,Core,earth-system-discipline
-recommended,Recommended,earth-system-discipline
+Name,Description
+core,Core
+recommended,Recommended
diff --git a/topic-hierarchy/earth-system-discipline.csv b/topic-hierarchy/earth-system-discipline.csv
diff --git a/topic-hierarchy/earth-system-discipline/atmospheric-composition/index.csv b/topic-hierarchy/earth-system-discipline/atmospheric-composition/index.csv
@@ -0,0 +1,2 @@
+Name,Description
+experimental,Experimental topics
diff --git a/topic-hierarchy/earth-system-discipline/climate/index.csv b/topic-hierarchy/earth-system-discipline/climate/index.csv
@@ -0,0 +1,2 @@
+Name,Description
+experimental,Experimental topics
diff --git a/topic-hierarchy/earth-system-discipline/cryosphere/index.csv b/topic-hierarchy/earth-system-discipline/cryosphere/index.csv
@@ -0,0 +1,2 @@
+Name,Description
+experimental,Experimental topics
diff --git a/topic-hierarchy/earth-system-discipline/hydrology/index.csv b/topic-hierarchy/earth-system-discipline/hydrology/index.csv
@@ -0,0 +1,2 @@
+Name,Description
+experimental,Experimental topics
diff --git a/topic-hierarchy/earth-system-discipline/index.csv b/topic-hierarchy/earth-system-discipline/index.csv
@@ -0,0 +1,8 @@
+Name,Description
+weather,Weather
+climate,Climate
+hydrology,Hydrology
+atmospheric-composition,Atmospheric Composition
+cryosphere,Cryosphere
+ocean,Ocean
+space-weather,Space Weather
diff --git a/topic-hierarchy/earth-system-discipline/ocean/index.csv b/topic-hierarchy/earth-system-discipline/ocean/index.csv
@@ -0,0 +1,2 @@
+Name,Description
+experimental,Experimental topics
diff --git a/topic-hierarchy/earth-system-discipline/space-weather/index.csv b/topic-hierarchy/earth-system-discipline/space-weather/index.csv
@@ -0,0 +1,2 @@
+Name,Description
+experimental,Experimental topics
diff --git a/topic-hierarchy/earth-system-discipline/weather/index.csv b/topic-hierarchy/earth-system-discipline/weather/index.csv
@@ -0,0 +1,3 @@
+Name,Description
+surface-based-observations,Surface based observations
+experimental,Experimental topics
diff --git a/topic-hierarchy/earth-system-discipline/weather/surface-based-observations/index.csv b/topic-hierarchy/earth-system-discipline/weather/surface-based-observations/index.csv
@@ -0,0 +1,3 @@
+Name,Description
+synop,Surface observations
+temp,Upper air observations
diff --git a/topic-hierarchy/hydrology.csv b/topic-hierarchy/hydrology.csv
diff --git a/topic-hierarchy/notification-type.csv b/topic-hierarchy/notification-type.csv
@@ -1,3 +1,3 @@
-Name,Description,Child
-data,Data,data-policy
-metadata,Metadata,data-policy
+Name,Description
+data,Data
+metadata,Metadata
diff --git a/topic-hierarchy/ocean.csv b/topic-hierarchy/ocean.csv
diff --git a/topic-hierarchy/system.csv b/topic-hierarchy/system.csv
@@ -1,2 +1,2 @@
-Name,Description,Child
-wis2,WIS2 topic hierarchy,centre-id
+Name,Description
+wis2,WIS2 topic hierarchy
diff --git a/topic-hierarchy/version.csv b/topic-hierarchy/version.csv
@@ -1,2 +1,2 @@
-Name,Description,Child
-a,version a,system
+Name,Description
+a,version a
diff --git a/topic-hierarchy/weather-surface-based-observations-report-type.csv b/topic-hierarchy/weather-surface-based-observations-report-type.csv
diff --git a/topic-hierarchy/weather.csv b/topic-hierarchy/weather.csv
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,3 +2,4 @@ @@
     *.html
     *.pdf
     .DS_Store
+    topic-hierarchy/earth-system-discipline.csv
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Name,Description
		experimental,Experimental topics