From 514e95f23b7897c9d39b8c32800269f624b1a800 Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Sat, 27 May 2023 11:41:11 +0200 Subject: [PATCH] Pick up CAF patch for ISO 8601, add new btest --- caf | 2 +- .../web-socket.timestamp/recv.recv.out | 23 ++ tests/btest/web-socket/timestamp.py | 271 ++++++++++++++++++ tests/cpp/internal/json_type_mapper.cc | 22 +- 4 files changed, 311 insertions(+), 7 deletions(-) create mode 100644 tests/btest/Baseline/web-socket.timestamp/recv.recv.out create mode 100644 tests/btest/web-socket/timestamp.py diff --git a/caf b/caf index 571c4d19b..d3cfa1fea 160000 --- a/caf +++ b/caf @@ -1 +1 @@ -Subproject commit 571c4d19ba6b72328e04e0452da9618cd0c9d3f3 +Subproject commit d3cfa1fea5447278c45c6a930e654d1cd51b7d57 diff --git a/tests/btest/Baseline/web-socket.timestamp/recv.recv.out b/tests/btest/Baseline/web-socket.timestamp/recv.recv.out new file mode 100644 index 000000000..60e8a4d8c --- /dev/null +++ b/tests/btest/Baseline/web-socket.timestamp/recv.recv.out @@ -0,0 +1,23 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +got 21 items +OK: 2020-02-02T02:02:02+00:00 +OK: 2020-02-02T02:02:02.1+00:00 +OK: 2020-02-02T02:02:02.12+00:00 +OK: 2020-02-02T02:02:02.123+00:00 +OK: 2020-02-02T02:02:02.1234+00:00 +OK: 2020-02-02T02:02:02.12345+00:00 +OK: 2020-02-02T02:02:02.123456+00:00 +OK: 2020-02-02T02:02:02+02:00 +OK: 2020-02-02T02:02:02.1+02:00 +OK: 2020-02-02T02:02:02.12+02:00 +OK: 2020-02-02T02:02:02.123+02:00 +OK: 2020-02-02T02:02:02.1234+02:00 +OK: 2020-02-02T02:02:02.12345+02:00 +OK: 2020-02-02T02:02:02.123456+02:00 +OK: 2020-02-02T02:02:02-02:00 +OK: 2020-02-02T02:02:02.1-02:00 +OK: 2020-02-02T02:02:02.12-02:00 +OK: 2020-02-02T02:02:02.123-02:00 +OK: 2020-02-02T02:02:02.1234-02:00 +OK: 2020-02-02T02:02:02.12345-02:00 +OK: 2020-02-02T02:02:02.123456-02:00 diff --git a/tests/btest/web-socket/timestamp.py b/tests/btest/web-socket/timestamp.py new file mode 100644 index 000000000..774a33c5d --- /dev/null +++ b/tests/btest/web-socket/timestamp.py @@ -0,0 +1,271 @@ +# @TEST-GROUP: web-socket +# +# @TEST-PORT: BROKER_WEB_SOCKET_PORT +# +# @TEST-EXEC: btest-bg-run node "broker-node --config-file=../node.cfg" +# @TEST-EXEC: btest-bg-run recv "python3 ../recv.py >recv.out" +# @TEST-EXEC: $SCRIPTS/wait-for-file recv/ready 15 || (btest-bg-wait -k 1 && false) +# +# @TEST-EXEC: btest-bg-run send "python3 ../send.py" +# +# @TEST-EXEC: $SCRIPTS/wait-for-file recv/done 30 || (btest-bg-wait -k 1 && false) +# @TEST-EXEC: btest-diff recv/recv.out +# +# @TEST-EXEC: btest-bg-wait -k 1 + +@TEST-START-FILE node.cfg + +broker { + disable-ssl = true +} +topics = ["/test"] +verbose = true + +@TEST-END-FILE + +@TEST-START-FILE recv.py + +import asyncio, websockets, os, time, json, sys + + +from datetime import datetime + +ws_port = os.environ['BROKER_WEB_SOCKET_PORT'].split('/')[0] + +ws_url = f'ws://localhost:{ws_port}/v1/messages/json' + +# Expected timestamps in the message. +timestamps = [ + # using UTC (note: 'Z' is not supported on older Python versions) + '2020-02-02T02:02:02+00:00', + '2020-02-02T02:02:02.1+00:00', + '2020-02-02T02:02:02.12+00:00', + '2020-02-02T02:02:02.123+00:00', + '2020-02-02T02:02:02.1234+00:00', + '2020-02-02T02:02:02.12345+00:00', + '2020-02-02T02:02:02.123456+00:00', + # using positive offset from UTC + '2020-02-02T02:02:02+02:00', + '2020-02-02T02:02:02.1+02:00', + '2020-02-02T02:02:02.12+02:00', + '2020-02-02T02:02:02.123+02:00', + '2020-02-02T02:02:02.1234+02:00', + '2020-02-02T02:02:02.12345+02:00', + '2020-02-02T02:02:02.123456+02:00', + # using negative offset from UTC + '2020-02-02T02:02:02-02:00', + '2020-02-02T02:02:02.1-02:00', + '2020-02-02T02:02:02.12-02:00', + '2020-02-02T02:02:02.123-02:00', + '2020-02-02T02:02:02.1234-02:00', + '2020-02-02T02:02:02.12345-02:00', + '2020-02-02T02:02:02.123456-02:00', +] + +# tells btest we're done by writing a file +def write_done_file(): + with open('done', 'w') as f: + f.write('done') + +def check_msg(msg): + try: + # iterate over the message and compare it to the expected values + if msg['@data-type'] != 'vector': + raise RuntimeError(f'unexpected data type for the message: {msg["@data-type"]}') + items = msg['data'] + num_items = len(items) + if num_items != len(timestamps): + raise RuntimeError(f'unexpected number of items: {len(items)} != {len(timestamps)}') + print(f'got {num_items} items') + for i in range(num_items): + item = items[i] + if item['@data-type'] != 'timestamp': + raise RuntimeError(f'unexpected data type at index {i}: {item["@data-type"]} != timestamp') + dt1 = datetime.fromisoformat(item['data']) + dt2 = datetime.fromisoformat(timestamps[i]) + if dt1 != dt2: + raise RuntimeError(f'unexpected timestamp at index {i}: {item["data"]} != {timestamps[i]}') + print(f'OK: {timestamps[i]}') + except Exception as e: + print(f'*** {e}') + +async def do_run(): + # Try up to 30 times. + connected = False + for i in range(30): + try: + ws = await websockets.connect(ws_url) + connected = True + # send filter and wait for ack + await ws.send('["/test"]') + ack_json = await ws.recv() + ack = json.loads(ack_json) + if not 'type' in ack or ack['type'] != 'ack': + print('*** unexpected ACK from server:') + print(ack_json) + sys.exit() + # tell btest to start the sender now + with open('ready', 'w') as f: + f.write('ready') + # get and verify the message + msg_json = await ws.recv() + msg = json.loads(msg_json) + check_msg(msg) + # tell btest we're done + write_done_file() + await ws.close() + sys.exit() + except: + if not connected: + print(f'failed to connect to {ws_url}, try again', file=sys.stderr) + time.sleep(1) + else: + write_done_file() + sys.exit() + +loop = asyncio.get_event_loop() +loop.run_until_complete(do_run()) + +@TEST-END-FILE + +@TEST-START-FILE send.py + +import asyncio, websockets, os, json, sys + +ws_port = os.environ['BROKER_WEB_SOCKET_PORT'].split('/')[0] + +ws_url = f'ws://localhost:{ws_port}/v1/messages/json' + +# Message with timestamps using the local time zone. +msg = { + 'type': 'data-message', + 'topic': '/test', + '@data-type': "vector", + 'data': [ + # UTC timestamps + + # timestamp without fractional seconds + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02Z" + }, + # timestamp with fractional seconds (1 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.1Z" + }, + # timestamp with fractional seconds (2 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.12Z" + }, + # timestamp with fractional seconds (3 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.123Z" + }, + # timestamp with fractional seconds (4 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.1234Z" + }, + # timestamp with fractional seconds (5 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.12345Z" + }, + # timestamp with fractional seconds (6 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.123456Z" + }, + + # timestamps that use a positive offset from UTC. + + # timestamp without fractional seconds + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02+02" + }, + # timestamp with fractional seconds (1 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.1+0200" + }, + # timestamp with fractional seconds (2 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.12+02:00" + }, + # timestamp with fractional seconds (3 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.123+02:00" + }, + # timestamp with fractional seconds (4 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.1234+02:00" + }, + # timestamp with fractional seconds (5 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.12345+02:00" + }, + # timestamp with fractional seconds (6 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.123456+02:00" + }, + + # timestamps that use a negative offset from UTC. + + # timestamp without fractional seconds + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02-02" + }, + # timestamp with fractional seconds (1 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.1-0200" + }, + # timestamp with fractional seconds (2 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.12-02:00" + }, + # timestamp with fractional seconds (3 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.123-02:00" + }, + # timestamp with fractional seconds (4 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.1234-02:00" + }, + # timestamp with fractional seconds (5 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.12345-02:00" + }, + # timestamp with fractional seconds (6 digit) + { + '@data-type': "timestamp", + "data": "2020-02-02T02:02:02.123456-02:00" + }, + ], +} + +async def do_run(): + async with websockets.connect(ws_url) as ws: + await ws.send('[]') + await ws.recv() # wait for ACK + await ws.send(json.dumps(msg)) + await ws.close() + +loop = asyncio.get_event_loop() +loop.run_until_complete(do_run()) + +@TEST-END-FILE diff --git a/tests/cpp/internal/json_type_mapper.cc b/tests/cpp/internal/json_type_mapper.cc index 10adb7834..682cf92b7 100644 --- a/tests/cpp/internal/json_type_mapper.cc +++ b/tests/cpp/internal/json_type_mapper.cc @@ -6,6 +6,7 @@ #include #include +#include using namespace broker; @@ -56,7 +57,7 @@ constexpr caf::string_view json = R"_({ }, { "@data-type": "timestamp", - "data": "2022-04-10T16:07:00.000" + "data": "2022-04-10T16:07:00Z" }, { "@data-type": "timespan", @@ -134,7 +135,7 @@ data_message native() { xs.emplace_back(dummy_addr_v6); xs.emplace_back(subnet{dummy_addr_v4, 24}); xs.emplace_back(port{8080, port::protocol::tcp}); - xs.emplace_back(timestamp_from_string("2022-04-10T16:07:00.000")); + xs.emplace_back(timestamp_from_string("2022-04-10T16:07:00Z")); xs.emplace_back(timespan{23s}); xs.emplace_back(enum_value{"foo"s}); xs.emplace_back(set{data{1}, data{2}, data{3}}); @@ -172,8 +173,17 @@ TEST(the JSON mapper enables custom type names in JSON output) { writer.mapper(&mapper); auto msg = native(); auto decorator = decorated(msg); - if (CHECK(writer.apply(decorator))) - CHECK_EQ(writer.str(), json); - else - auto str = to_string(writer.str()); + if (CHECK(writer.apply(decorator))) { + auto expected_json = to_string(json); + // Give the timestamp in the expected output a roundtrip through the + // chrono library to normalize the format. + auto utc = "2022-04-10T16:07:00Z"s; + auto parsed = caf::chrono::datetime::from_string(utc); + REQUIRE(parsed); + auto ts = parsed->to_local_time(); + auto normalized = caf::chrono::to_string(ts); + MESSAGE("normalized timestamp from " << utc << " to " << normalized); + caf::replace_all(expected_json, utc, normalized); + CHECK_EQ(writer.str(), expected_json); + } }