From 4ebef3ff948ae0d24d64a6f773727b8f26a6592e Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Mon, 15 Aug 2022 17:32:02 +0300 Subject: [PATCH 01/11] fixing the tests instead closes #48 --- test_commands.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test_commands.py b/test_commands.py index fcf3c97..61c85e3 100644 --- a/test_commands.py +++ b/test_commands.py @@ -248,16 +248,13 @@ def testTDigestQuantile(self): # assert min min/max have same result as quantile 0 and 1 self.assertEqual( float(rb.tdigestMax('tDigest')), - float(rb.tdigestQuantile('tDigest', 1.0)), + float(rb.tdigestQuantile('tDigest', 99.99000000000001)[0]), ) self.assertEqual( float(rb.tdigestMin('tDigest')), - float(rb.tdigestQuantile('tDigest', 0.0)), + float(rb.tdigestQuantile('tDigest', 0.01)[0]), ) - self.assertAlmostEqual(1.0, float(rb.tdigestQuantile('tDigest', 0.01)), 2) - self.assertAlmostEqual(99.0, float(rb.tdigestQuantile('tDigest', 0.99)), 2) - def testTDigestCdf(self): self.assertTrue(rb.tdigestCreate('tDigest', 100)) # insert data-points into sketch From b56d0705bcc152be1e2a5dd4c90192b08410b1b0 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 16 Aug 2022 14:14:28 +0300 Subject: [PATCH 02/11] xonsh issue? --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8bd13ce..a19fb92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,6 +50,7 @@ commands: - run: name: install tox dependencies command: | + sudo apt remove -y python3-virtualenv pip install --user --upgrade pip virtualenv pip install --user --quiet -r .circleci/circle_requirements.txt From 1d7a6d446704cca156d954096fa0f149cc0b9b34 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 16 Aug 2022 14:18:33 +0300 Subject: [PATCH 03/11] disable poetry venv control --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a19fb92..92649b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,6 @@ commands: - run: name: install tox dependencies command: | - sudo apt remove -y python3-virtualenv pip install --user --upgrade pip virtualenv pip install --user --quiet -r .circleci/circle_requirements.txt @@ -59,6 +58,7 @@ commands: command: | # https://github.com/python-poetry/poetry/issues/4210 poetry config experimental.new-installer false + poetry config virtualenvs.create fals poetry build - run: From 1518f3dae3be0f8cf7904fa3cfb263267654233d Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 16 Aug 2022 14:23:53 +0300 Subject: [PATCH 04/11] fixed spelling --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92649b3..3e64543 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,7 @@ commands: command: | # https://github.com/python-poetry/poetry/issues/4210 poetry config experimental.new-installer false - poetry config virtualenvs.create fals + poetry config virtualenvs.create false poetry build - run: From 528b6eaa7d6a667082f663cd594688a31f31118b Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 16 Aug 2022 14:34:40 +0300 Subject: [PATCH 05/11] trying to debug poetry issue --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e64543..567d0e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ commands: - run: name: install tox dependencies command: | - pip install --user --upgrade pip virtualenv + pip install --user --upgrade pip virtualenv poetry pip install --user --quiet -r .circleci/circle_requirements.txt - run: @@ -84,7 +84,7 @@ jobs: default: "latest" docker: - image: circleci/python:<> - - image: redislabs/rebloom:edge + - image: redislabs/redismod:edge steps: - build_and_test From 0b55aee8d9c33664e65c7dbc98fc48f09c726421 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 1 Sep 2022 13:54:02 +0300 Subject: [PATCH 06/11] removing tdigest --- .circleci/config.yml | 1 + redisbloom/client.py | 117 ------------------------------------------- test_commands.py | 56 --------------------- 3 files changed, 1 insertion(+), 173 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 567d0e4..d271675 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,6 +50,7 @@ commands: - run: name: install tox dependencies command: | + pip uninstall poetry pip install --user --upgrade pip virtualenv poetry pip install --user --quiet -r .circleci/circle_requirements.txt diff --git a/redisbloom/client.py b/redisbloom/client.py index f888863..a091cf9 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -71,26 +71,6 @@ def __init__(self, args): self.decay = response['decay'] -class TDigestInfo(object): - compression = None - capacity = None - mergedNodes = None - unmergedNodes = None - mergedWeight = None - unmergedWeight = None - totalCompressions = None - - def __init__(self, args): - response = dict(zip(map(nativestr, args[::2]), args[1::2])) - self.compression = response['Compression'] - self.capacity = response['Capacity'] - self.mergedNodes = response['Merged nodes'] - self.unmergedNodes = response['Unmerged nodes'] - self.mergedWeight = response['Merged weight'] - self.unmergedWeight = response['Unmerged weight'] - self.totalCompressions = response['Total compressions'] - - def spaceHolder(response): return response @@ -118,7 +98,6 @@ class Client(Redis): # changed from StrictRedis - CF for Cuckoo Filter - CMS for Count-Min Sketch - TOPK for TopK Data Structure - - TDIGEST for estimate rank statistics """ BF_RESERVE = 'BF.RESERVE' @@ -157,16 +136,6 @@ class Client(Redis): # changed from StrictRedis TOPK_LIST = 'TOPK.LIST' TOPK_INFO = 'TOPK.INFO' - TDIGEST_CREATE = 'TDIGEST.CREATE' - TDIGEST_RESET = 'TDIGEST.RESET' - TDIGEST_ADD = 'TDIGEST.ADD' - TDIGEST_MERGE = 'TDIGEST.MERGE' - TDIGEST_CDF = 'TDIGEST.CDF' - TDIGEST_QUANTILE = 'TDIGEST.QUANTILE' - TDIGEST_MIN = 'TDIGEST.MIN' - TDIGEST_MAX = 'TDIGEST.MAX' - TDIGEST_INFO = 'TDIGEST.INFO' - def __init__(self, *args, **kwargs): """ Creates a new RedisBloom client. @@ -212,15 +181,6 @@ def __init__(self, *args, **kwargs): self.TOPK_LIST: parseToList, self.TOPK_INFO: TopKInfo, - self.TDIGEST_CREATE: bool_ok, - # self.TDIGEST_RESET: bool_ok, - # self.TDIGEST_ADD: spaceHolder, - # self.TDIGEST_MERGE: spaceHolder, - # self.TDIGEST_CDF: spaceHolder, - # self.TDIGEST_QUANTILE: spaceHolder, - # self.TDIGEST_MIN: spaceHolder, - # self.TDIGEST_MAX: spaceHolder, - self.TDIGEST_INFO: TDigestInfo, } for k, v in six.iteritems(MODULE_CALLBACKS): self.set_response_callback(k, v) @@ -613,83 +573,6 @@ def topkInfo(self, key): return self.execute_command(self.TOPK_INFO, key) -################## T-Digest Functions ###################### - - def tdigestCreate(self, key, compression): - """" - Allocate the memory and initialize the t-digest. - """ - params = [key, compression] - - return self.execute_command(self.TDIGEST_CREATE, *params) - - def tdigestReset(self, key): - """ - Reset the sketch ``key`` to zero - empty out the sketch and re-initialize it. - """ - - return self.execute_command(self.TDIGEST_RESET, key) - - def tdigestAdd(self, key, values, weights): - """ - Adds one or more samples (value with weight) to a sketch ``key``. - Both ``values`` and ``weights`` are lists. - Example - tdigestAdd('A', [1500.0], [1.0]) - """ - params = [key] - self.appendValuesAndWeights(params, values, weights) - - return self.execute_command(self.TDIGEST_ADD, *params) - - def tdigestMerge(self, toKey, fromKey): - """ - Merges all of the values from 'fromKey' to 'toKey' sketch. - """ - params = [toKey, fromKey] - - return self.execute_command(self.TDIGEST_MERGE, *params) - - def tdigestMin(self, key): - """ - Returns minimum value from the sketch ``key``. - Will return DBL_MAX if the sketch is empty. - """ - - return self.execute_command(self.TDIGEST_MIN, key) - - def tdigestMax(self, key): - """ - Returns maximum value from the sketch ``key``. - Will return DBL_MIN if the sketch is empty. - """ - - return self.execute_command(self.TDIGEST_MAX, key) - - def tdigestQuantile(self, key, quantile): - """ - Returns double value estimate of the cutoff such that a specified fraction of the data added - to this TDigest would be less than or equal to the cutoff. - """ - params = [key, quantile] - - return self.execute_command(self.TDIGEST_QUANTILE, *params) - - def tdigestCdf(self, key, value): - """ - Returns double fraction of all points added which are <= value. - """ - params = [key, value] - - return self.execute_command(self.TDIGEST_CDF, *params) - - def tdigestInfo(self, key): - """ - Returns Compression, Capacity, Merged Nodes, Unmerged Nodes, Merged Weight, Unmerged Weight - and Total Compressions. - """ - - return self.execute_command(self.TDIGEST_INFO, key) - def pipeline(self, transaction=True, shard_hint=None): """ Return a new pipeline object that can queue multiple commands for diff --git a/test_commands.py b/test_commands.py index 61c85e3..58d7602 100644 --- a/test_commands.py +++ b/test_commands.py @@ -34,7 +34,6 @@ def testCreate(self): self.assertTrue(rb.cmsInitByDim('cmsDim', 100, 5)) self.assertTrue(rb.cmsInitByProb('cmsProb', 0.01, 0.01)) self.assertTrue(rb.topkReserve('topk', 5, 100, 5, 0.9)) - self.assertTrue(rb.tdigestCreate('tDigest', 100)) ################### Test Bloom Filter ################### def testBFAdd(self): @@ -208,61 +207,6 @@ def testTopK(self): self.assertEqual(3, info.depth) self.assertAlmostEqual(0.9, float(info.decay)) - ################### Test T-Digest ################### - def testTDigestReset(self): - self.assertTrue(rb.tdigestCreate('tDigest', 10)) - # reset on empty histogram - self.assertTrue(rb.tdigestReset('tDigest')) - # insert data-points into sketch - self.assertTrue(rb.tdigestAdd('tDigest', list(range(10)), [1.0] * 10)) - - self.assertTrue(rb.tdigestReset('tDigest')) - # assert we have 0 unmerged nodes - self.assertEqual(0, rb.tdigestInfo('tDigest').unmergedNodes) - - def testTDigestMerge(self): - self.assertTrue(rb.tdigestCreate('to-tDigest', 10)) - self.assertTrue(rb.tdigestCreate('from-tDigest', 10)) - # insert data-points into sketch - self.assertTrue(rb.tdigestAdd('from-tDigest', [1.0] * 10, [1.0] * 10)) - self.assertTrue(rb.tdigestAdd('to-tDigest', [2.0] * 10, [10.0] * 10)) - # merge from-tdigest into to-tdigest - self.assertTrue(rb.tdigestMerge('to-tDigest', 'from-tDigest')) - # we should now have 110 weight on to-histogram - info = rb.tdigestInfo('to-tDigest') - total_weight_to = float(info.mergedWeight) + float(info.unmergedWeight) - self.assertEqual(110, total_weight_to) - - def testTDigestMinMax(self): - self.assertTrue(rb.tdigestCreate('tDigest', 100)) - # insert data-points into sketch - self.assertTrue(rb.tdigestAdd('tDigest', [1, 2, 3], [1.0] * 3)) - # min/max - self.assertEqual(3, float(rb.tdigestMax('tDigest'))) - self.assertEqual(1, float(rb.tdigestMin('tDigest'))) - - def testTDigestQuantile(self): - self.assertTrue(rb.tdigestCreate('tDigest', 500)) - # insert data-points into sketch - self.assertTrue(rb.tdigestAdd('tDigest', list([x * 0.01 for x in range(1, 10000)]), [1.0] * 10000)) - # assert min min/max have same result as quantile 0 and 1 - self.assertEqual( - float(rb.tdigestMax('tDigest')), - float(rb.tdigestQuantile('tDigest', 99.99000000000001)[0]), - ) - self.assertEqual( - float(rb.tdigestMin('tDigest')), - float(rb.tdigestQuantile('tDigest', 0.01)[0]), - ) - - def testTDigestCdf(self): - self.assertTrue(rb.tdigestCreate('tDigest', 100)) - # insert data-points into sketch - self.assertTrue(rb.tdigestAdd('tDigest', list(range(1, 10)), [1.0] * 10)) - - self.assertAlmostEqual(0.1, float(rb.tdigestCdf('tDigest', 1.0)), 1) - self.assertAlmostEqual(0.9, float(rb.tdigestCdf('tDigest', 9.0)), 1) - def test_pipeline(self): pipeline = rb.pipeline() From 0d9c382688b64de5eb9cea8435b7236768b1aa5f Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 1 Sep 2022 14:06:21 +0300 Subject: [PATCH 07/11] -y on the uninstall --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d271675..f3a222c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ commands: - run: name: install tox dependencies command: | - pip uninstall poetry + pip uninstall -y poetry pip install --user --upgrade pip virtualenv poetry pip install --user --quiet -r .circleci/circle_requirements.txt From 5ad3405c187c491fa95464e899fcc0dbf2913170 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 1 Sep 2022 14:07:15 +0300 Subject: [PATCH 08/11] versioning --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e6d2d0f..788813a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisbloom" -version = "0.4.1" +version = "0.5.0" description = "RedisBloom Python Client" authors = ["Redis "] license = "BSD-3-Clause" From f6eb62cd25b198ebebefe771cd0555825a384452 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 1 Sep 2022 14:08:38 +0300 Subject: [PATCH 09/11] sudo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3a222c..0b960df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ commands: - run: name: install tox dependencies command: | - pip uninstall -y poetry + sudo pip uninstall -y poetry pip install --user --upgrade pip virtualenv poetry pip install --user --quiet -r .circleci/circle_requirements.txt From c08ab329eb4825b841ea2dc3eb4c5f74c75b72b7 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 1 Sep 2022 14:25:27 +0300 Subject: [PATCH 10/11] more --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b960df..1c1c89b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ commands: - run: name: install tox dependencies command: | - sudo pip uninstall -y poetry + sudo pip uninstall -y poetry virtualenv pip install --user --upgrade pip virtualenv poetry pip install --user --quiet -r .circleci/circle_requirements.txt From c96c09e54949a31d3e218a2f632b927c796a3131 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Sun, 4 Sep 2022 11:05:36 +0300 Subject: [PATCH 11/11] fixed linting after merge --- redisbloom/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/redisbloom/__init__.py b/redisbloom/__init__.py index a914243..102bc65 100644 --- a/redisbloom/__init__.py +++ b/redisbloom/__init__.py @@ -1,4 +1,8 @@ from warnings import warn -warn("Please upgrade to redis-py (https://pypi.org/project/redis/) " -"This library is deprecated, and all features have been merged into redis-py.", -DeprecationWarning, stacklevel=2) + +warn( + "Please upgrade to redis-py (https://pypi.org/project/redis/) " + "This library is deprecated, and all features have been merged into redis-py.", + DeprecationWarning, + stacklevel=2, +)