Merge branch 'master' into remove/ngsiv1-location-metadata

.github/workflows/compliance.yml

@@ -2,13 +2,16 @@ name: Compliance Tests
 
 on:
   push:
+    branches:
+      - master
   pull_request:
     branches:
       - master
+      - '!checkvalgrind**'
 
 jobs:
   compliance:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/functional.yml

@@ -2,37 +2,40 @@ name: Functional Tests
 
 on:
   push:
+    branches:
+      - master
   pull_request:
     branches:
       - master
+      - '!checkvalgrind**'
 
 concurrency: 
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-   TEST_IMAGE_NAME: fiware/orion-ci:deb
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   functional:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     continue-on-error: true
 
     services:
       mongodb:
-        image: mongo:4.4
+        image: mongo:6.0
         ports:
           - 27017:27017
 
       mosquitto:
         # Needed by MQTT notification tests
-        image: eclipse-mosquitto:1.6.7
+        image: eclipse-mosquitto:1.6.15
         ports:
         - 1883:1883
 
       mosquitto-extra:
         # Needed by MQTT notification tests (multi broker)
-        image: eclipse-mosquitto:1.6.7
+        image: eclipse-mosquitto:1.6.15
         ports:
         - 1884:1883
 

.github/workflows/publishimage.yml

@@ -19,7 +19,7 @@ env:
 jobs:
   deploy-release:
 
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     if: github.event_name == 'push'
 
     steps:

.github/workflows/unit.yml

@@ -2,24 +2,27 @@ name: Unit Tests
 
 on:
   push:
+     branches:
+      - master
   pull_request:
     branches:
       - master
+      - '!checkvalgrind**'
 
 concurrency: 
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-  TEST_IMAGE_NAME: fiware/orion-ci:deb
+  TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   unit:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
 
     services:
       mongodb:
-        image: mongo:4.4
+        image: mongo:6.0
         ports:
           - 27017:27017
 

.github/workflows/valgrind-nocache.yml

@@ -0,0 +1,61 @@
+# Same as valgrind.yml but with CB_NO_CACHE=ON activated
+# We use a separate .yml file because it is better in the case of re-launching due to glitches
+
+# Valgrind action is a heavy process that can take arround 50 minutes to finish. Thus, we don't want
+# this to be executed in every pull request but only intentionally on specific branches. In particular,
+# the action is configured so it will run in branches starting with 'checkvalgrind'
+
+name: Valgrind Tests Nocache
+
+on:
+  push:
+    branches:
+      - checkvalgrind**
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
+
+jobs:
+  valgrind:
+    runs-on: ubuntu-22.04
+    continue-on-error: true
+
+    services:
+      mongodb:
+        image: mongo:6.0
+        ports:
+          - 27017:27017
+
+      mosquitto:
+        # Needed by MQTT notification tests
+        image: eclipse-mosquitto:1.6.15
+        ports:
+        - 1883:1883
+
+      mosquitto-extra:
+        # Needed by MQTT notification tests (multi broker)
+        image: eclipse-mosquitto:1.6.15
+        ports:
+        - 1884:1883
+
+    name: valgrind - ${{ matrix.payload.name }}
+
+    strategy:
+      matrix:
+        payload:
+          - { name: 'batch 1 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=0 -e FT_TO_IX=300'}
+          - { name: 'batch 2 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=301 -e FT_TO_IX=600'}
+          - { name: 'batch 3 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=601 -e FT_TO_IX=900'}
+          - { name: 'batch 4 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=901 -e FT_TO_IX=1200'}
+          - { name: 'batch 5 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=1201'}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run valgrind test
+        run: |
+          docker run  --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind

.github/workflows/valgrind.yml

@@ -0,0 +1,58 @@
+# Valgrind action is a heavy process that can take arround 50 minutes to finish. Thus, we don't want
+# this to be executed in every pull request but only intentionally on specific branches. In particular,
+# the action is configured so it will run in branches starting with 'checkvalgrind'
+
+name: Valgrind Tests
+
+on:
+  push:
+    branches:
+      - checkvalgrind**
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
+
+jobs:
+  valgrind:
+    runs-on: ubuntu-22.04
+    continue-on-error: true
+
+    services:
+      mongodb:
+        image: mongo:6.0
+        ports:
+          - 27017:27017
+
+      mosquitto:
+        # Needed by MQTT notification tests
+        image: eclipse-mosquitto:1.6.15
+        ports:
+        - 1883:1883
+
+      mosquitto-extra:
+        # Needed by MQTT notification tests (multi broker)
+        image: eclipse-mosquitto:1.6.15
+        ports:
+        - 1884:1883
+
+    name: valgrind - ${{ matrix.payload.name }}
+
+    strategy:
+      matrix:
+        payload:
+          - { name: 'batch 1 - cache', range: '-e FT_FROM_IX=0 -e FT_TO_IX=300'}
+          - { name: 'batch 2 - cache', range: '-e FT_FROM_IX=301 -e FT_TO_IX=600'}
+          - { name: 'batch 3 - cache', range: '-e FT_FROM_IX=601 -e FT_TO_IX=900'}
+          - { name: 'batch 4 - cache', range: '-e FT_FROM_IX=901 -e FT_TO_IX=1200'}
+          - { name: 'batch 5 - cache', range: '-e FT_FROM_IX=1201'}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run valgrind test
+        run: |
+          docker run  --privileged --network host -t --rm ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind

.readthedocs.yml

@@ -3,7 +3,11 @@ version: 2
 mkdocs:
   configuration: mkdocs.yml
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+
 python:
-  version: 3.8
   install:
     - requirements: doc/requirements.txt

CHANGES_NEXT_RELEASE

@@ -1,8 +1,13 @@
-- Add: json field in httpCustom and mqttCustom subscriptions (#2560)
-- Add: ${service}, ${servicePath} and ${authToken} macros in custom notifications (#4159)
-- Fix: conditions.alterationTypes not working properly when conditions.attributes is used in entityUpdate case (#1494, reopened)
-- Fix: GET /v2/types and GET /v2/types/{type} wrong attribute types result (#4216)
-- Upgrade Debian version from 11.2 to 11.3 in Dockerfile
-- Deprecated: GET /v2 operation
+- Add: servicePath field to builtin attributes (#2877)
+- Add: notification.mqtt.retain and notification.mqttCustom.retain flag for MQTT retain in notifications (#4388)
+- Fix: correctly detect JSON attribute and metadata value changes in subscription triggering logic (#4211, #4434, #643)
+- Fix: DateTime and geo:json types were not supported in custom notifications using ngsi patching (#4435)
+- Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated)
+- Fix: improve error traces (#4387)
+- Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794)
+- Fix: improve logs in MongoDB query logic
 - Remove: deprecated NGSIv1 location metadata to specify entity geo-location
-- Remove: RPM package stuff
+- Upgrade Debian version from 11.6 to 12.1 in Dockerfile
+- Hardening: upgrade libmongoc dependency from 1.23.1 to 1.24.3
+- Reference MongoDB version changed from 4.4 to 6.0
+- Reference distribution changed from Debian 11 to Debian 12

CMakeLists.txt

@@ -171,6 +171,12 @@ endif (${CMAKE_BUILD_TYPE} STREQUAL DEBUG)
 #    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable")
 #endif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
 
+# Enables some some #ifdef in the code for compiling in old system
+# FIXME: cleanup OLD_SSL_VERSION_FORMAT stuff after consolidating the change to Debian 12+
+IF((${DISTRO} MATCHES "Debian_11.*") OR (${DISTRO} MATCHES "Alpine_3.16.*"))
+  ADD_DEFINITIONS(-DOLD_SSL_VERSION_FORMAT)
+ENDIF()
+
 #
 # Libraries
 #
@@ -220,8 +226,8 @@ SET (BOOST_MT
 
 # See http://mongoc.org/libmongoc/current/tutorial.html#cmake
 # This find_package() command provides the mongo::mongoc_static used in
-# SET for common static libs. We use 1.17.4 as reference version.
-find_package (mongoc-1.0 1.17.4 EXACT)
+# SET for common static libs. We use 1.24.3 as reference version.
+find_package (mongoc-1.0 1.24.3 EXACT)
 
 # Static libs common to contextBroker and unitTest binaries
 SET (COMMON_STATIC_LIBS

Changelog

@@ -1,3 +1,65 @@
+3.10.0 (June 12th, 2023)
+
+- Add: option to change deprecate log setting in the REST management API
+- Fix: decimal rounding in some cases was generating wrong JSON in query responses and notifications (#4346)
+- Fix: pagination support in request query forwarding to Context Providers (#4149, #847, #1647)
+- Fix: count option not working when georel=near was in use (500 error was returned)
+- Fix: maxFailsLimit didn't work in some cases when update subscription is done just after cache refresh (only when csubs cache is enabled)
+- Fix: conditions.notifyOnMetadataChange false not working if condition.attrs is empty
+- Deprecate: `geo:json`, `geo:line`, `geo:box` and `geo:polygon` attribute types (use `geo:json` instead)
+- Remove: all NGSIv1 operations (deprecated in Orion 2.0.0) except the following ones
+    - PUT /v1/contextEntities/{id}
+    - DELETE /v1/contextEntities/{id}
+    - GET /v1/contextEntities/{id}/attributes/{name}
+    - POST /v1/updateContext
+    - POST /NGSI10/updateContext
+    - POST /v1/queryContext
+    - POST /NGSI10/queryContext
+
+3.9.0 (June 2th, 2023)
+
+- Add: new simplifiedNormalized and simplifiedKeyvalues for attrsFormat (#4286)
+- Add: subject.condition.notifyOnMetadataChange field to subscriptions, so only-metadata changes doesn't trigger notifications (#3727)
+- Add: support for update operators ($set, $addToSet, etc.) in entity creation and replace (#3814, reopnened)
+- Add: -mqttTimeout (env var ORION_MQTT_TIMEOUT) for MQTT broker connection timeout
+- Add: -disableNgsiv1 (env var ORION_DISABLE_NGSIV1) to disable NGSIv1 API requests
+- Add: -logDeprecate (env var ORION_LOG_DEPRECATE) to log deprecate usages at WARN level
+- Fix: alterationType with entityDelete not working if condition.attrs is empty (#4326)
+- Fix: $set/$unset update operator not working properly when the key has dots (#4315)
+- Fix: queue workers deadlock when MQTT broker was not responding to connection attemp
+- Fix: Ngsiv2-AttrsFormat header set to "normalized" in NGSI patching custom notification
+- Fix: memory access potential problem in GET /log/trace request when tracelevel 255 is enabled
+- Fix: Orion should return error and refuse to start when port is negative (#3875)
+- Fix: subscription autodisable based on maxFailsLimit not working when fails does not happen within the same csubs cache refresh cycle
+- Fix: incorrectly sum stale failsCounter from database when subscription has started to work again (only when csubs cache is enabled)
+- Fix: MQTT disconnection when MQTT publish fail (so re-connection is forced at next MQTT notification)
+- Fix: lineMaxSize and infoPayloadMaxSize in the log admin REST API (#3707)
+- Fix: check for service database name too long at cache sync logic (#2848)
+- Hardening: upgrade microhttpd dependency from 0.9.73 to 0.9.76
+
+3.8.0 (January 12th, 2023)
+
+- Add: json field in httpCustom and mqttCustom subscriptions to implement JSON-based payloads in notifications (#2560)
+- Add: ngsi field in httpCustom and mqttCustom subscriptions to implement NGSI payload patching in notifications (#4085)
+- Add: ${service}, ${servicePath} and ${authToken} macros in custom notifications (#4159)
+- Add: "Authorization" in CORS allowed headers (#4249)
+- Add: requestLegacy counters in GET /statistics response
+- Fix: conditions.alterationTypes not working properly when conditions.attributes is used in entityUpdate case (#1494, reopened)
+- Fix: GET /v2/types and GET /v2/types/{type} wrong attribute types result (#4216)
+- Fix: reset measuring_interval_in_secs correctly reset upon DELETE /statistics (#4233)
+- Fix: version request correctly recorded in statistics API (in "/version".GET instead than in separated versionRequests counter) (#4233)
+- Fix: GET /v2/registrations/{regId} was wrongly returning 200 OK instaed of 404 Not Found in case of non existing registration (#4235)
+- Fix: GET /statistics counters requests now use URLs intead of symbolic names
+- Fix: replace $where operator by $expr in subscription query logic in the -noCache case (#4148)
+- Upgrade Debian version from 11.2 to 11.6 in Dockerfile
+- Hardening: upgrade microhttpd dependency from 0.9.70 to 0.9.73
+- Hardening: upgrade libmosquitto dependency from 2.0.12 to 2.0.15
+- Hardening: upgrade libmongoc dependency from 1.17.4 to 1.23.1
+- Deprecated: GET /v2 operation
+- Remove: rendundant routes in administrative REST API not aligned with documentation
+- Remove: /ngsi9 URL paths (deprecated since Orion 1.2.0)
+- Remove: RPM package stuff
+
 3.7.0 (May 26th, 2022)
 
 - Add: conditions.alterationTypes subscription fuctionality (#1494)