From 9a99b93a51eadacfb7dffbb2243f0aca5d463d6b Mon Sep 17 00:00:00 2001 From: winton <365504029@qq.com> Date: Wed, 2 Sep 2020 20:26:19 +0800 Subject: [PATCH 1/5] upgrade python package version and fix the test assert. --- .travis.yml | 4 +++- README.rst | 1 + requirements.txt | 12 ++++++------ setup.py | 1 + ttseries/ts/base.py | 1 - ttseries/ts/hash.py | 6 +++--- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4db93b2..c493f53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,11 @@ matrix: - python: 3.7 sudo: required dist: xenial + - python: 3.8 + sudo: required services: - - redis-server + - redis compiler: - clang diff --git a/README.rst b/README.rst index 8811323..14bd95f 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,7 @@ High performance engine to store Time series data in Redis. |travis| |appveyor| |codecov| |codacy| |requirements| |docs| |pypi| |status| |pyversion| |download| +Redis Version required: >=5.0 TT-series is based on redis sorted sets to store the time-series data, `Sorted set`_ store scores with unique numbers under a single key, but it has a weakness to store records, only unique members are allowed diff --git a/requirements.txt b/requirements.txt index b9773a8..0cafb4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -msgpack>=0.6.1 -numpy>=1.16.0 -python-dateutil>=2.7.5 -redis>=3.0.0 -hiredis>=1.0.0 -pandas>=0.24.0 +msgpack>=1.0.0 +numpy>=1.19.1 +python-dateutil>=2.8.1 +redis>=3.5.3 +hiredis>=1.1.0 +pandas>=1.1.1 diff --git a/setup.py b/setup.py index 4939001..4605af2 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ def find_version(*file_paths): "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Topic :: Software Development", "Topic :: Software Development :: Libraries" ], diff --git a/ttseries/ts/base.py b/ttseries/ts/base.py index c5ce71d..1d7c2c0 100644 --- a/ttseries/ts/base.py +++ b/ttseries/ts/base.py @@ -55,7 +55,6 @@ def __init__(self, redis_client, max_length=100000, transaction=True, raise SerializerError("Serializer class must inherit from " "ttseries.serializers.BaseSerializer abstract class") - @property @functools.lru_cache(maxsize=4096) def client(self): diff --git a/ttseries/ts/hash.py b/ttseries/ts/hash.py index f55f7e4..2afd39e 100644 --- a/ttseries/ts/hash.py +++ b/ttseries/ts/hash.py @@ -82,7 +82,7 @@ def add(self, name: str, timestamp: float, data) -> bool: def pipe_func(_pipe): # trans function _pipe.zadd(name, {key_id: timestamp}) - _pipe.hmset(hash_key, dumps_dict) # APPL:second:HASH, {1:value} + _pipe.hset(hash_key, mapping=dumps_dict) # APPL:second:HASH, {1:value} watch_keys = (name, hash_key) # APPL:SECOND , APPL:second:HASH return self.transaction_pipe(pipe_func, watch_keys) @@ -236,7 +236,7 @@ def add_many(self, name, array: list, chunks_size=2000): def pipe_func(_pipe): _pipe.zadd(name, timestamp_ids) - _pipe.hmset(hash_key, ids_values) + _pipe.hset(hash_key, mapping=ids_values) self.transaction_pipe(pipe_func, watch_keys=(name, hash_key)) @@ -260,4 +260,4 @@ def iter(self, name): for timestamp_pairs, hash_pairs in itertools.zip_longest(self.client.zscan_iter(name=name), self.client.hscan_iter(name=hash_key)): - yield (timestamp_pairs[1], self._serializer.loads(hash_pairs[1])) + yield timestamp_pairs[1], self._serializer.loads(hash_pairs[1]) From cbf010dcc200e6842400d1673c200fa88d5157fd Mon Sep 17 00:00:00 2001 From: winton <365504029@qq.com> Date: Wed, 2 Sep 2020 20:35:49 +0800 Subject: [PATCH 2/5] add appveyor python3.8 --- .appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 727cd4d..9f2aab8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -34,6 +34,10 @@ environment: PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8" + PYTHON_ARCH: "64" + init: # If there is a newer build queued for the same PR, cancel this one. # The AppVeyor 'rollout builds' option is supposed to serve the same From 5c0940cde25fcd5d18a30c4192e3cb1d388bab33 Mon Sep 17 00:00:00 2001 From: winton <365504029@qq.com> Date: Wed, 2 Sep 2020 22:37:58 +0800 Subject: [PATCH 3/5] update benchmark report. --- benchmark/report.txt | 151 ++++++++++++++++++-------------- tests/test_pandas_timeseries.py | 20 ++--- 2 files changed, 90 insertions(+), 81 deletions(-) diff --git a/benchmark/report.txt b/benchmark/report.txt index 7ddc9b0..cad2aff 100644 --- a/benchmark/report.txt +++ b/benchmark/report.txt @@ -1,81 +1,96 @@ -MacOSX 10.13.1 +MacOSX 10.15.6 +Redis server v=6.0.7 sha=00000000:0 malloc=libc bits=64 build=161937e85b34847d +CUP info: 2.3 GHz Quad-Core Intel Core i5 +Memory info: 16 GB 2133 MHz LPDDR3 -Redis server v=4.0.11 sha=00000000:0 malloc=libc bits=64 build=13f90e3a88f770eb - -CUP info: Intel(R) Core(TM) i5-5257U CPU @ 2.70GHz - -Memory info: 8 GB 1867 MHz DDR3 +----------------------------------------------------------------------------------------------------- benchmark 'hash': 6 tests ----------------------------------------------------------------------------------------------------- +Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_get_hash_timeseries_without_serializer[1000] 2.6256 (1.0) 10.5870 (1.0) 3.3110 (1.0) 0.6019 (1.0) 3.2416 (1.0) 0.6173 (1.0) 29;1 302.0259 (1.0) 252 1 +test_add_hash_timeseries_without_serializer[data0] 15.2429 (5.81) 22.0359 (2.08) 17.9701 (5.43) 1.5785 (2.62) 17.5871 (5.43) 2.3596 (3.82) 14;0 55.6479 (0.18) 47 1 +test_get_hash_timeseries_without_serializer[10000] 29.6847 (11.31) 35.1273 (3.32) 31.5479 (9.53) 1.2235 (2.03) 31.3810 (9.68) 1.4854 (2.41) 8;1 31.6978 (0.10) 35 1 +test_add_hash_timeseries_without_serializer[data1] 124.7450 (47.51) 129.3957 (12.22) 127.0947 (38.39) 1.8014 (2.99) 126.9014 (39.15) 3.3998 (5.51) 3;0 7.8682 (0.03) 8 1 +test_get_hash_timeseries_without_serializer[100000] 335.4058 (127.74) 350.7834 (33.13) 342.2519 (103.37) 5.5520 (9.22) 342.0749 (105.52) 5.5120 (8.93) 2;0 2.9218 (0.01) 5 1 +test_add_hash_timeseries_without_serializer[data2] 1,317.3542 (501.72) 1,334.6582 (126.07) 1,324.2972 (399.97) 7.2400 (12.03) 1,321.0067 (407.51) 11.3038 (18.31) 1;0 0.7551 (0.00) 5 1 +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------------------------------------------------------------------------------------ benchmark 'hash': 6 tests ------------------------------------------------------------------------------------------------------ -Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_get_hash_timeseries_without_serializer[1000] 4.4350 (1.0) 10.5665 (1.0) 4.8492 (1.0) 0.5067 (1.0) 4.7864 (1.0) 0.1569 (1.0) 5;13 206.2175 (1.0) 166 1 -test_add_hash_timeseries_without_serializer[data0] 24.6785 (5.56) 31.0453 (2.94) 26.5885 (5.48) 1.5298 (3.02) 25.8275 (5.40) 1.6980 (10.82) 7;2 37.6102 (0.18) 33 1 -test_get_hash_timeseries_without_serializer[10000] 43.8963 (9.90) 48.3694 (4.58) 45.8016 (9.45) 1.2979 (2.56) 45.4267 (9.49) 1.5060 (9.60) 6;0 21.8333 (0.11) 21 1 -test_add_hash_timeseries_without_serializer[data1] 200.5232 (45.21) 212.0145 (20.06) 208.4725 (42.99) 4.5918 (9.06) 209.6233 (43.80) 4.3626 (27.80) 1;0 4.7968 (0.02) 5 1 -test_get_hash_timeseries_without_serializer[100000] 473.2930 (106.72) 500.3805 (47.36) 490.8784 (101.23) 11.5141 (22.72) 496.7709 (103.79) 17.0357 (108.56) 1;0 2.0372 (0.01) 5 1 -test_add_hash_timeseries_without_serializer[data2] 2,009.2282 (453.04) 2,092.3974 (198.02) 2,057.5223 (424.30) 30.7886 (60.76) 2,067.3346 (431.92) 33.3587 (212.57) 2;0 0.4860 (0.00) 5 1 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +--------------------------------------------------------------------------------------------------- benchmark 'numpy': 15 tests ---------------------------------------------------------------------------------------------------- +Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +test_get_numpy_timeseries_serializer[1000] 2.9352 (1.0) 9.6446 (1.0) 3.3580 (1.0) 0.4705 (1.0) 3.2918 (1.0) 0.4293 (1.0) 19;5 297.8005 (1.0) 292 1 +test_add_numpy_timeseries_serializer[2000-data0] 13.2728 (4.52) 17.8204 (1.85) 14.8387 (4.42) 1.0421 (2.22) 14.5082 (4.41) 1.4100 (3.28) 14;2 67.3911 (0.23) 62 1 +test_add_numpy_timeseries_serializer[5000-data0] 13.3696 (4.55) 16.9950 (1.76) 14.5271 (4.33) 0.8195 (1.74) 14.2985 (4.34) 0.7954 (1.85) 15;5 68.8368 (0.23) 62 1 +test_add_numpy_timeseries_serializer[8000-data0] 13.5088 (4.60) 29.6237 (3.07) 15.2296 (4.54) 2.4866 (5.29) 14.3626 (4.36) 1.4949 (3.48) 5;5 65.6616 (0.22) 61 1 +test_add_numpy_timeseries_serializer[10000-data0] 13.6008 (4.63) 21.2600 (2.20) 14.9162 (4.44) 1.1573 (2.46) 14.7046 (4.47) 1.1904 (2.77) 8;1 67.0410 (0.23) 64 1 +test_get_numpy_timeseries_serializer[10000] 30.1409 (10.27) 36.1494 (3.75) 32.0976 (9.56) 1.3077 (2.78) 32.0863 (9.75) 1.3887 (3.23) 6;2 31.1550 (0.10) 31 1 +test_add_numpy_timeseries_serializer[10000-data1] 102.4548 (34.91) 108.7597 (11.28) 106.0746 (31.59) 2.1098 (4.48) 105.4843 (32.04) 3.3716 (7.85) 3;0 9.4273 (0.03) 9 1 +test_add_numpy_timeseries_serializer[8000-data1] 103.8854 (35.39) 115.0508 (11.93) 109.1482 (32.50) 3.5541 (7.55) 109.5009 (33.27) 4.9303 (11.48) 3;0 9.1619 (0.03) 9 1 +test_add_numpy_timeseries_serializer[5000-data1] 105.0935 (35.80) 113.4881 (11.77) 109.8735 (32.72) 2.2518 (4.79) 110.2387 (33.49) 1.4029 (3.27) 2;2 9.1014 (0.03) 9 1 +test_add_numpy_timeseries_serializer[2000-data1] 108.7711 (37.06) 117.4456 (12.18) 111.4173 (33.18) 2.6030 (5.53) 110.7545 (33.65) 2.4857 (5.79) 2;1 8.9753 (0.03) 9 1 +test_get_numpy_timeseries_serializer[100000] 342.2626 (116.61) 355.3574 (36.85) 349.5864 (104.11) 5.1349 (10.91) 350.4173 (106.45) 7.7201 (17.98) 2;0 2.8605 (0.01) 5 1 +test_add_numpy_timeseries_serializer[8000-data2] 1,091.8922 (372.00) 1,119.7147 (116.10) 1,109.2682 (330.34) 10.9087 (23.19) 1,112.1773 (337.87) 14.3347 (33.39) 1;0 0.9015 (0.00) 5 1 +test_add_numpy_timeseries_serializer[10000-data2] 1,099.0443 (374.43) 1,121.2712 (116.26) 1,112.1634 (331.20) 8.8465 (18.80) 1,113.3974 (338.24) 13.2492 (30.86) 2;0 0.8991 (0.00) 5 1 +test_add_numpy_timeseries_serializer[5000-data2] 1,109.8792 (378.13) 1,125.3821 (116.69) 1,118.2274 (333.01) 6.0528 (12.87) 1,117.3429 (339.44) 8.9318 (20.80) 2;0 0.8943 (0.00) 5 1 +test_add_numpy_timeseries_serializer[2000-data2] 1,139.4673 (388.21) 1,149.4636 (119.18) 1,144.6348 (340.87) 3.6465 (7.75) 1,144.6365 (347.73) 4.3698 (10.18) 2;0 0.8736 (0.00) 5 1 +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ --------------------------------------------------------------------------------------------------- benchmark 'numpy': 6 tests ------------------------------------------------------------------------------------------------- -Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_get_numpy_timeseries_serializer[1000] 4.4665 (1.0) 7.0042 (1.0) 4.6508 (1.0) 0.3553 (1.0) 4.5373 (1.0) 0.1124 (1.0) 16;26 215.0184 (1.0) 216 1 -test_add_numpy_timeseries_serializer[data0] 19.1978 (4.30) 25.6876 (3.67) 20.4825 (4.40) 1.3838 (3.89) 19.7469 (4.35) 2.0779 (18.49) 10;1 48.8222 (0.23) 46 1 -test_get_numpy_timeseries_serializer[10000] 44.1389 (9.88) 53.2824 (7.61) 46.8347 (10.07) 2.1741 (6.12) 46.7463 (10.30) 2.5385 (22.59) 4;1 21.3517 (0.10) 22 1 -test_add_numpy_timeseries_serializer[data1] 161.0182 (36.05) 175.5067 (25.06) 166.9047 (35.89) 5.3720 (15.12) 166.2073 (36.63) 7.1573 (63.69) 2;0 5.9914 (0.03) 6 1 -test_get_numpy_timeseries_serializer[100000] 465.5352 (104.23) 486.7678 (69.50) 478.3363 (102.85) 8.4060 (23.66) 481.4245 (106.10) 11.7887 (104.91) 2;0 2.0906 (0.01) 5 1 -test_add_numpy_timeseries_serializer[data2] 1,685.7103 (377.41) 1,715.2885 (244.89) 1,695.0657 (364.47) 12.3900 (34.87) 1,689.1056 (372.27) 16.2690 (144.78) 1;0 0.5899 (0.00) 5 1 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------- benchmark 'numpy_dtype': 6 tests ------------------------------------------------------------------------------------------------ +Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_get_numpy_dtype_timeseries_serializer[1000] 2.2732 (1.0) 4.4903 (1.0) 2.7652 (1.0) 0.3153 (1.0) 2.7260 (1.0) 0.3827 (1.0) 126;5 361.6438 (1.0) 358 1 +test_add_numpy_dtype_timeseries_serializer[data0] 14.8888 (6.55) 18.3727 (4.09) 16.2245 (5.87) 0.8766 (2.78) 16.0842 (5.90) 1.2174 (3.18) 17;0 61.6351 (0.17) 55 1 +test_get_numpy_dtype_timeseries_serializer[10000] 25.5184 (11.23) 29.5570 (6.58) 27.0019 (9.77) 1.0469 (3.32) 27.2346 (9.99) 1.7675 (4.62) 15;0 37.0344 (0.10) 37 1 +test_add_numpy_dtype_timeseries_serializer[data1] 122.6080 (53.94) 130.5211 (29.07) 125.2118 (45.28) 3.0083 (9.54) 123.9695 (45.48) 4.8539 (12.68) 2;0 7.9865 (0.02) 8 1 +test_get_numpy_dtype_timeseries_serializer[100000] 270.1021 (118.82) 282.8432 (62.99) 275.3513 (99.58) 4.9370 (15.66) 274.6607 (100.76) 6.8795 (17.98) 2;0 3.6317 (0.01) 5 1 +test_add_numpy_dtype_timeseries_serializer[data2] 1,289.8747 (567.43) 1,307.2837 (291.14) 1,294.5477 (468.17) 7.3273 (23.24) 1,290.7276 (473.49) 7.1827 (18.77) 1;0 0.7725 (0.00) 5 1 +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- --------------------------------------------------------------------------------------------------- benchmark 'numpy_dtype': 6 tests ------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- benchmark 'pandas': 6 tests ---------------------------------------------------------------------------------------------------- Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_get_numpy_dtype_timeseries_serializer[1000] 4.3871 (1.0) 7.4806 (1.0) 4.8257 (1.0) 0.3724 (1.0) 4.7172 (1.0) 0.3136 (1.0) 25;15 207.2245 (1.0) 220 1 -test_add_numpy_dtype_timeseries_serializer[data0] 20.6232 (4.70) 25.0124 (3.34) 22.3432 (4.63) 1.1147 (2.99) 22.0991 (4.68) 1.5264 (4.87) 15;0 44.7564 (0.22) 40 1 -test_get_numpy_dtype_timeseries_serializer[10000] 44.5654 (10.16) 54.5620 (7.29) 47.6574 (9.88) 2.3018 (6.18) 47.9187 (10.16) 2.7512 (8.77) 5;1 20.9831 (0.10) 23 1 -test_add_numpy_dtype_timeseries_serializer[data1] 178.8879 (40.78) 189.8636 (25.38) 183.1170 (37.95) 4.1012 (11.01) 183.0642 (38.81) 5.7542 (18.35) 2;0 5.4610 (0.03) 6 1 -test_get_numpy_dtype_timeseries_serializer[100000] 450.7964 (102.76) 466.8610 (62.41) 455.8997 (94.47) 6.4614 (17.35) 455.0152 (96.46) 6.9506 (22.16) 1;0 2.1935 (0.01) 5 1 -test_add_numpy_dtype_timeseries_serializer[data2] 1,825.9456 (416.21) 1,899.1509 (253.88) 1,865.3157 (386.54) 28.0709 (75.37) 1,863.3795 (395.02) 40.1780 (128.10) 2;0 0.5361 (0.00) 5 1 +test_get_pandas_timeseries_serializer[1000] 4.5124 (1.0) 11.8311 (1.0) 5.2631 (1.0) 0.7258 (1.0) 5.2362 (1.0) 0.5132 (1.0) 13;7 190.0006 (1.0) 190 1 +test_add_pandas_timeseries_serializer[data_frame0] 14.4637 (3.21) 17.7414 (1.50) 15.9567 (3.03) 0.8385 (1.16) 15.9730 (3.05) 1.0734 (2.09) 20;0 62.6695 (0.33) 56 1 +test_get_pandas_timeseries_serializer[10000] 34.3676 (7.62) 40.2221 (3.40) 37.3181 (7.09) 1.5255 (2.10) 37.3078 (7.12) 1.9788 (3.86) 9;0 26.7966 (0.14) 26 1 +test_add_pandas_timeseries_serializer[data_frame1] 111.0704 (24.61) 116.8684 (9.88) 113.4682 (21.56) 1.8327 (2.52) 113.5223 (21.68) 2.4917 (4.85) 2;0 8.8130 (0.05) 9 1 +test_get_pandas_timeseries_serializer[100000] 364.3174 (80.74) 372.8958 (31.52) 368.9080 (70.09) 3.1291 (4.31) 369.2600 (70.52) 3.6575 (7.13) 2;0 2.7107 (0.01) 5 1 +test_add_pandas_timeseries_serializer[data_frame2] 1,136.2153 (251.80) 1,170.6054 (98.94) 1,153.6286 (219.19) 16.6651 (22.96) 1,153.1152 (220.22) 32.7553 (63.82) 2;0 0.8668 (0.00) 5 1 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- benchmark 'pandas': 6 tests ---------------------------------------------------------------------------------------------------- -Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_get_pandas_timeseries_serializer[1000] 11.4218 (1.0) 14.9347 (1.0) 11.9195 (1.0) 0.6343 (1.0) 11.7208 (1.0) 0.3907 (1.0) 6;6 83.8961 (1.0) 60 1 -test_add_pandas_timeseries_serializer[data_frame0] 21.3574 (1.87) 26.9966 (1.81) 22.9289 (1.92) 1.2584 (1.98) 22.6122 (1.93) 1.5636 (4.00) 10;1 43.6132 (0.52) 34 1 -test_get_pandas_timeseries_serializer[10000] 94.1123 (8.24) 100.1321 (6.70) 96.5160 (8.10) 2.1287 (3.36) 95.5540 (8.15) 3.1339 (8.02) 4;0 10.3610 (0.12) 11 1 -test_add_pandas_timeseries_serializer[data_frame1] 167.8461 (14.70) 176.6065 (11.83) 172.4146 (14.46) 2.8150 (4.44) 172.4536 (14.71) 1.4706 (3.76) 2;2 5.8000 (0.07) 6 1 -test_get_pandas_timeseries_serializer[100000] 916.7732 (80.27) 951.1308 (63.69) 928.7658 (77.92) 13.1874 (20.79) 926.2771 (79.03) 12.6065 (32.27) 1;0 1.0767 (0.01) 5 1 -test_add_pandas_timeseries_serializer[data_frame2] 1,681.4176 (147.21) 1,702.3345 (113.99) 1,695.9075 (142.28) 8.5214 (13.43) 1,698.7762 (144.94) 9.6634 (24.73) 1;0 0.5897 (0.01) 5 1 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - -------------------------------------------------------------------------------------------------- benchmark 'simple': 6 tests -------------------------------------------------------------------------------------------------- -Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_get_simple_timeseries_serializer[1000] 4.0993 (1.0) 6.3579 (1.0) 4.2549 (1.0) 0.3173 (1.0) 4.1748 (1.0) 0.0767 (1.0) 13;25 235.0210 (1.0) 236 1 -test_simple_timeseries_serializer[data0] 15.5233 (3.79) 18.7691 (2.95) 16.0602 (3.77) 0.7391 (2.33) 15.7729 (3.78) 0.3892 (5.07) 6;9 62.2658 (0.26) 49 1 -test_get_simple_timeseries_serializer[10000] 40.2922 (9.83) 45.5551 (7.17) 42.2976 (9.94) 1.5767 (4.97) 42.0968 (10.08) 2.9071 (37.89) 7;0 23.6420 (0.10) 23 1 -test_simple_timeseries_serializer[data1] 123.0189 (30.01) 130.6148 (20.54) 127.2522 (29.91) 2.6393 (8.32) 127.3503 (30.50) 3.8050 (49.59) 4;0 7.8584 (0.03) 8 1 -test_get_simple_timeseries_serializer[100000] 451.5924 (110.16) 462.2357 (72.70) 456.7458 (107.34) 4.2398 (13.36) 456.3889 (109.32) 6.7797 (88.36) 2;0 2.1894 (0.01) 5 1 -test_simple_timeseries_serializer[data2] 1,249.8994 (304.90) 1,281.5087 (201.56) 1,270.5288 (298.60) 13.3574 (42.10) 1,277.0945 (305.90) 19.5062 (254.22) 1;0 0.7871 (0.00) 5 1 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-------------------------------------------------------------------------------------------- benchmark 'simple': 6 tests --------------------------------------------------------------------------------------------- +Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_get_simple_timeseries_serializer[1000] 2.3125 (1.0) 4.5171 (1.0) 2.6695 (1.0) 0.2565 (1.0) 2.6683 (1.0) 0.3429 (1.0) 83;5 374.6012 (1.0) 348 1 +test_simple_timeseries_serializer[data0] 10.0840 (4.36) 12.2605 (2.71) 10.7884 (4.04) 0.5523 (2.15) 10.6266 (3.98) 0.8210 (2.39) 24;0 92.6920 (0.25) 75 1 +test_get_simple_timeseries_serializer[10000] 24.3219 (10.52) 30.8969 (6.84) 26.2222 (9.82) 1.2356 (4.82) 25.9283 (9.72) 1.3807 (4.03) 8;1 38.1356 (0.10) 38 1 +test_simple_timeseries_serializer[data1] 74.8520 (32.37) 80.8781 (17.90) 78.0492 (29.24) 1.8855 (7.35) 77.6339 (29.09) 2.7923 (8.14) 5;0 12.8124 (0.03) 13 1 +test_get_simple_timeseries_serializer[100000] 283.1227 (122.43) 290.1165 (64.23) 286.0083 (107.14) 3.3191 (12.94) 284.3419 (106.56) 5.9895 (17.47) 1;0 3.4964 (0.01) 5 1 +test_simple_timeseries_serializer[data2] 803.7080 (347.55) 818.4337 (181.19) 809.8323 (303.36) 5.6824 (22.16) 808.7663 (303.10) 7.8621 (22.93) 2;0 1.2348 (0.00) 5 1 +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- --------------------------------------------------------------------------------------------------- benchmark 'simple_dumpy': 9 tests --------------------------------------------------------------------------------------------------- -Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_get_simple_timeseries_dumpy_serializer[1000] 2.0313 (1.0) 5.9386 (1.0) 2.1214 (1.0) 0.2348 (1.0) 2.0751 (1.0) 0.0681 (1.0) 12;23 471.3886 (1.0) 343 1 -test_iter_simple_timeseries_dumpy_serializer[1000] 9.3127 (4.58) 14.6176 (2.46) 10.2992 (4.85) 0.8230 (3.51) 10.1973 (4.91) 0.4740 (6.96) 12;5 97.0953 (0.21) 90 1 -test_add_simple_timeseries_without_serializer[data0] 13.6854 (6.74) 16.7171 (2.81) 14.3248 (6.75) 0.7340 (3.13) 14.0707 (6.78) 0.5299 (7.79) 6;5 69.8090 (0.15) 49 1 -test_get_simple_timeseries_dumpy_serializer[10000] 19.3463 (9.52) 28.9985 (4.88) 20.5967 (9.71) 1.7749 (7.56) 19.9034 (9.59) 1.1651 (17.12) 7;6 48.5515 (0.10) 51 1 -test_iter_simple_timeseries_dumpy_serializer[10000] 95.6766 (47.10) 108.8663 (18.33) 102.9287 (48.52) 4.1955 (17.87) 103.0636 (49.67) 2.9695 (43.64) 4;2 9.7155 (0.02) 10 1 -test_add_simple_timeseries_without_serializer[data1] 106.6902 (52.52) 114.1583 (19.22) 109.1101 (51.43) 2.3515 (10.02) 108.1667 (52.13) 2.3720 (34.86) 2;1 9.1651 (0.02) 9 1 -test_get_simple_timeseries_dumpy_serializer[100000] 209.0582 (102.92) 221.9169 (37.37) 215.5528 (101.61) 5.3471 (22.78) 217.3072 (104.72) 8.7553 (128.66) 2;0 4.6392 (0.01) 5 1 -test_iter_simple_timeseries_dumpy_serializer[100000] 981.1015 (482.99) 1,217.6887 (205.05) 1,137.4765 (536.19) 99.2629 (422.84) 1,192.1250 (574.50) 136.4182 (>1000.0) 1;0 0.8791 (0.00) 5 1 -test_add_simple_timeseries_without_serializer[data2] 1,088.4091 (535.81) 1,102.2183 (185.60) 1,096.4635 (516.86) 5.6505 (24.07) 1,097.5518 (528.92) 9.0448 (132.91) 2;0 0.9120 (0.00) 5 1 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------- benchmark 'simple_dumpy': 18 tests ------------------------------------------------------------------------------------------------- +Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_get_simple_timeseries_dumpy_serializer[1000] 1.4043 (1.0) 11.2237 (1.0) 1.7579 (1.0) 0.4548 (1.0) 1.7308 (1.0) 0.1753 (1.0) 5;7 568.8582 (1.0) 502 1 +test_iter_simple_timeseries_dumpy_serializer[1000] 8.3600 (5.95) 20.7781 (1.85) 8.8494 (5.03) 1.3562 (2.98) 8.5908 (4.96) 0.1795 (1.02) 6;8 113.0016 (0.20) 101 1 +test_add_simple_timeseries_without_serializer[6000-data0] 9.0454 (6.44) 13.4399 (1.20) 10.1377 (5.77) 0.7598 (1.67) 9.9892 (5.77) 0.8396 (4.79) 15;3 98.6415 (0.17) 77 1 +test_add_simple_timeseries_without_serializer[5000-data0] 9.0542 (6.45) 12.7317 (1.13) 9.9698 (5.67) 0.6751 (1.48) 9.7626 (5.64) 1.0086 (5.75) 28;1 100.3030 (0.18) 98 1 +test_add_simple_timeseries_without_serializer[10000-data0] 9.0875 (6.47) 13.3068 (1.19) 9.8086 (5.58) 0.6728 (1.48) 9.6418 (5.57) 0.6296 (3.59) 13;4 101.9509 (0.18) 84 1 +test_add_simple_timeseries_without_serializer[2000-data0] 9.4443 (6.73) 13.2528 (1.18) 10.1618 (5.78) 0.7289 (1.60) 9.9535 (5.75) 0.7860 (4.48) 7;2 98.4078 (0.17) 58 1 +test_get_simple_timeseries_dumpy_serializer[10000] 14.2320 (10.13) 17.4892 (1.56) 16.0053 (9.10) 0.8282 (1.82) 16.0806 (9.29) 1.1931 (6.81) 19;0 62.4792 (0.11) 57 1 +test_add_simple_timeseries_without_serializer[10000-data1] 60.7956 (43.29) 68.3448 (6.09) 63.7057 (36.24) 2.0631 (4.54) 63.4079 (36.63) 2.1494 (12.26) 4;1 15.6972 (0.03) 16 1 +test_add_simple_timeseries_without_serializer[5000-data1] 62.1602 (44.26) 69.6823 (6.21) 64.5361 (36.71) 1.8125 (3.99) 64.1849 (37.08) 1.8418 (10.50) 3;1 15.4952 (0.03) 15 1 +test_add_simple_timeseries_without_serializer[6000-data1] 62.9438 (44.82) 71.0727 (6.33) 65.8139 (37.44) 2.6894 (5.91) 64.9165 (37.51) 4.4054 (25.13) 5;0 15.1944 (0.03) 15 1 +test_add_simple_timeseries_without_serializer[2000-data1] 65.3024 (46.50) 69.6788 (6.21) 67.6320 (38.47) 1.2930 (2.84) 67.6027 (39.06) 0.8868 (5.06) 4;4 14.7859 (0.03) 14 1 +test_iter_simple_timeseries_dumpy_serializer[10000] 87.2666 (62.14) 101.2269 (9.02) 90.4564 (51.46) 3.8707 (8.51) 88.9062 (51.37) 2.7451 (15.66) 1;1 11.0551 (0.02) 12 1 +test_get_simple_timeseries_dumpy_serializer[100000] 169.4336 (120.65) 173.3395 (15.44) 171.3108 (97.45) 1.5406 (3.39) 171.2086 (98.92) 2.4161 (13.78) 2;0 5.8373 (0.01) 5 1 +test_add_simple_timeseries_without_serializer[10000-data2] 661.3510 (470.94) 665.9730 (59.34) 664.0895 (377.77) 1.9042 (4.19) 664.8400 (384.12) 2.9582 (16.87) 1;0 1.5058 (0.00) 5 1 +test_add_simple_timeseries_without_serializer[6000-data2] 670.6236 (477.54) 686.5039 (61.17) 679.3155 (386.43) 5.8659 (12.90) 679.6533 (392.68) 7.2595 (41.40) 2;0 1.4721 (0.00) 5 1 +test_add_simple_timeseries_without_serializer[5000-data2] 675.4889 (481.01) 781.3459 (69.62) 738.9950 (420.38) 56.0353 (123.21) 778.8847 (450.01) 101.2127 (577.26) 2;0 1.3532 (0.00) 5 1 +test_add_simple_timeseries_without_serializer[2000-data2] 710.7013 (506.08) 723.5439 (64.47) 715.5333 (407.04) 5.2834 (11.62) 714.2990 (412.69) 8.0394 (45.85) 1;0 1.3976 (0.00) 5 1 +test_iter_simple_timeseries_dumpy_serializer[100000] 926.9528 (660.07) 953.7220 (84.97) 935.2303 (532.01) 10.7458 (23.63) 932.9980 (539.05) 10.7434 (61.27) 1;0 1.0693 (0.00) 5 1 +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Legend: Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile. - OPS: Operations Per Second, computed as 1 / Mean \ No newline at end of file + OPS: Operations Per Second, computed as 1 / Mean diff --git a/tests/test_pandas_timeseries.py b/tests/test_pandas_timeseries.py index 8ba4405..432c079 100644 --- a/tests/test_pandas_timeseries.py +++ b/tests/test_pandas_timeseries.py @@ -16,14 +16,12 @@ def test_add(self): series_item = data_frame.iloc[0] - self.time_series.add(key, series_item) datetime_value = data_frame.index.to_pydatetime()[0] timestamp = datetime_value.timestamp() - result = self.time_series.get(key, timestamp) pandas.testing.assert_series_equal(series_item, result) @@ -35,20 +33,18 @@ def test_get_slice(self): self.time_series.add_many(key, data_frame) results_frame = self.time_series.get_slice(key) - - - pandas.testing.assert_frame_equal(results_frame,data_frame) + self.assertTrue(data_frame.equals(results_frame)) def test_iter(self): key = "AAPL:SECOND" data_frame = self.prepare_dataframe(10) self.time_series.add_many(key, data_frame) - new_data_frame = self.dataframe_empty(self.columns, - dtypes=self.dtypes) + new_data_frame = self.dataframe_empty(self.columns, dtypes=self.dtypes) for series in self.time_series.iter(key): new_data_frame = new_data_frame.append(series) - pandas.testing.assert_frame_equal(data_frame, new_data_frame) + + self.assertTrue(data_frame.equals(new_data_frame)) def test_add_exists_timestamp_assert_error(self): key = "AAPL:SECOND" @@ -66,7 +62,7 @@ def test_add_many_trim_data(self): self.time_series.add_many(key, data_frame2) results_frame = self.time_series.get_slice(key) - pandas.testing.assert_frame_equal(results_frame, data_frame2.iloc[10:]) + self.assertTrue(results_frame.equals(data_frame2.iloc[10:])) class RedisPandasTimeSeriesTest(unittest.TestCase, RedisPandasMixin): @@ -96,13 +92,11 @@ def prepare_dataframe(self, length, time_span=None): date_range = pandas.date_range(now, periods=length, freq="1min") - data_frame = pandas.DataFrame([i + 1 for i in range(len(date_range))], - index=date_range, columns=self.columns) + data_frame = pandas.DataFrame([i + 1 for i in range(len(date_range))], + index=date_range, columns=self.columns) data_frame.index.name = self.index_name return data_frame - - def dataframe_empty(self, columns, dtypes, index=None): data_frame = pandas.DataFrame(index=index) for column in columns: From 373bf2e117d4f7d9aea2b8b1899d872debbef7e4 Mon Sep 17 00:00:00 2001 From: winton <365504029@qq.com> Date: Wed, 2 Sep 2020 22:39:30 +0800 Subject: [PATCH 4/5] update version --- ttseries/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ttseries/__init__.py b/ttseries/__init__.py index 262da2a..05073a9 100644 --- a/ttseries/__init__.py +++ b/ttseries/__init__.py @@ -5,4 +5,4 @@ from .ts import RedisPandasTimeSeries from .serializers import BaseSerializer -__version__ = "0.2.1" +__version__ = "0.2.2" From 6651a3e42b6189f3e8b3ba52bf18e7852f667f22 Mon Sep 17 00:00:00 2001 From: winton <365504029@qq.com> Date: Thu, 3 Sep 2020 10:56:25 +0800 Subject: [PATCH 5/5] add zpopmin func --- pytest.ini | 1 - ttseries/ts/pandas.py | 3 +-- ttseries/ts/sample.py | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index 060ea5e..02ae4ea 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,7 +3,6 @@ ; To send tests to multiple CPUs ;addopts = -n2 -v --boxed -# todo support multi tests addopts = -v testpaths = tests \ No newline at end of file diff --git a/ttseries/ts/pandas.py b/ttseries/ts/pandas.py index 13b054e..e75be9c 100644 --- a/ttseries/ts/pandas.py +++ b/ttseries/ts/pandas.py @@ -149,8 +149,7 @@ def add(self, name, series): values = series.tolist() data = self._serializer.dumps(values) if self.length(name) == self.max_length: - # todo use 5.0 BZPOPMIN - self.client.zremrangebyrank(name, min=0, max=0) + self.client.zpopmin(name) return self.client.zadd(name, {data: timestamp}) else: raise RedisTimeSeriesError("Please check series Type or " diff --git a/ttseries/ts/sample.py b/ttseries/ts/sample.py index 27d8ce5..5c64ceb 100644 --- a/ttseries/ts/sample.py +++ b/ttseries/ts/sample.py @@ -40,8 +40,7 @@ def add(self, name: str, timestamp: float, data): if not self.exist_timestamp(name, timestamp): data = self._serializer.dumps(data) if self.length(name) == self.max_length: - # todo use 5.0 BZPOPMIN - self.client.zremrangebyrank(name, min=0, max=0) + self.client.zpopmin(name) return self.client.zadd(name, mapping={data: timestamp})