Round hll cardinality results to 10 decimal places (#112)

Author: hanefi

expected/cumulative_add_cardinality_correction.out

@@ -56,13 +56,13 @@ SELECT curr.recno,
   FROM test_msgfjqhm prev, test_msgfjqhm curr
  WHERE curr.recno > 1
    AND curr.recno = prev.recno + 1
-   AND curr.cardinality != 
-       hll_cardinality(hll_add(prev.union_compressed_multiset,
-                       hll_hashval(curr.raw_value)))
+   AND round(curr.cardinality::numeric, 10) != 
+       round(hll_cardinality(hll_add(prev.union_compressed_multiset,
+                             hll_hashval(curr.raw_value)))::numeric,
+             10)
  ORDER BY curr.recno;
- recno |   cardinality    | hll_cardinality  
--------+------------------+------------------
-  1792 | 4250.71186178904 | 4250.71186178904
-(1 row)
+ recno | cardinality | hll_cardinality 
+-------+-------------+-----------------
+(0 rows)
 
 DROP TABLE test_msgfjqhm;

expected/cumulative_add_comprehensive_promotion.out

@@ -56,27 +56,13 @@ SELECT curr.recno,
   FROM test_ptwysrqk prev, test_ptwysrqk curr
  WHERE curr.recno > 1
    AND curr.recno = prev.recno + 1
-   AND curr.cardinality != 
-       hll_cardinality(hll_add(prev.union_compressed_multiset,
-                       curr.raw_value::hll_hashval))
+   AND round(curr.cardinality::numeric, 10) != 
+       round(hll_cardinality(hll_add(prev.union_compressed_multiset,
+                             curr.raw_value::hll_hashval))::numeric,
+             10)
  ORDER BY curr.recno;
- recno |   cardinality    | hll_cardinality  
--------+------------------+------------------
-  4343 | 4250.71186178904 | 4250.71186178904
-  4344 | 4250.71186178904 | 4250.71186178904
-  4345 | 4250.71186178904 | 4250.71186178904
-  4346 | 4250.71186178904 | 4250.71186178904
-  4347 | 4250.71186178904 | 4250.71186178904
-  4348 | 4250.71186178904 | 4250.71186178904
-  4349 | 4250.71186178904 | 4250.71186178904
-  4350 | 4250.71186178904 | 4250.71186178904
-  4351 | 4250.71186178904 | 4250.71186178904
-  4352 | 4250.71186178904 | 4250.71186178904
-  4353 | 4250.71186178904 | 4250.71186178904
-  4354 | 4250.71186178904 | 4250.71186178904
-  4355 | 4250.71186178904 | 4250.71186178904
-  4356 | 4250.71186178904 | 4250.71186178904
-  4357 | 4250.71186178904 | 4250.71186178904
-(15 rows)
+ recno | cardinality | hll_cardinality 
+-------+-------------+-----------------
+(0 rows)
 
 DROP TABLE test_ptwysrqk;

expected/cumulative_union_sparse_promotion.out

@@ -51,7 +51,8 @@ SELECT recno,
        union_cardinality,
        hll_cardinality(union_compressed_multiset)
   FROM test_bsnvqefe
- WHERE union_cardinality != hll_cardinality(union_compressed_multiset)
+ WHERE round(union_cardinality::numeric, 10) !=
+       round(hll_cardinality(union_compressed_multiset)::numeric, 10)
  ORDER BY recno;
  recno | union_cardinality | hll_cardinality 
 -------+-------------------+-----------------
@@ -81,9 +82,10 @@ SELECT curr.recno,
   FROM test_bsnvqefe prev, test_bsnvqefe curr
  WHERE curr.recno > 1
    AND curr.recno = prev.recno + 1
-   AND curr.union_cardinality != 
-       hll_cardinality(hll_union(curr.compressed_multiset,
-                                 prev.union_compressed_multiset))
+   AND round(curr.union_cardinality::numeric, 10) != 
+       round(hll_cardinality(hll_union(curr.compressed_multiset,
+                                       prev.union_compressed_multiset))::numeric,
+            10)
  ORDER BY curr.recno;
  recno | union_cardinality | hll_cardinality 
 -------+-------------------+-----------------

sql/cumulative_add_cardinality_correction.sql

@@ -44,9 +44,10 @@ SELECT curr.recno,
   FROM test_msgfjqhm prev, test_msgfjqhm curr
  WHERE curr.recno > 1
    AND curr.recno = prev.recno + 1
-   AND curr.cardinality != 
-       hll_cardinality(hll_add(prev.union_compressed_multiset,
-                       hll_hashval(curr.raw_value)))
+   AND round(curr.cardinality::numeric, 10) != 
+       round(hll_cardinality(hll_add(prev.union_compressed_multiset,
+                             hll_hashval(curr.raw_value)))::numeric,
+             10)
  ORDER BY curr.recno;
 
 DROP TABLE test_msgfjqhm;

sql/cumulative_add_comprehensive_promotion.sql

@@ -44,9 +44,10 @@ SELECT curr.recno,
   FROM test_ptwysrqk prev, test_ptwysrqk curr
  WHERE curr.recno > 1
    AND curr.recno = prev.recno + 1
-   AND curr.cardinality != 
-       hll_cardinality(hll_add(prev.union_compressed_multiset,
-                       curr.raw_value::hll_hashval))
+   AND round(curr.cardinality::numeric, 10) != 
+       round(hll_cardinality(hll_add(prev.union_compressed_multiset,
+                             curr.raw_value::hll_hashval))::numeric,
+             10)
  ORDER BY curr.recno;
 
 DROP TABLE test_ptwysrqk;

sql/cumulative_union_sparse_promotion.sql

@@ -39,7 +39,8 @@ SELECT recno,
        union_cardinality,
        hll_cardinality(union_compressed_multiset)
   FROM test_bsnvqefe
- WHERE union_cardinality != hll_cardinality(union_compressed_multiset)
+ WHERE round(union_cardinality::numeric, 10) !=
+       round(hll_cardinality(union_compressed_multiset)::numeric, 10)
  ORDER BY recno;
 
 -- Test union of incremental multiset.
@@ -63,9 +64,10 @@ SELECT curr.recno,
   FROM test_bsnvqefe prev, test_bsnvqefe curr
  WHERE curr.recno > 1
    AND curr.recno = prev.recno + 1
-   AND curr.union_cardinality != 
-       hll_cardinality(hll_union(curr.compressed_multiset,
-                                 prev.union_compressed_multiset))
+   AND round(curr.union_cardinality::numeric, 10) != 
+       round(hll_cardinality(hll_union(curr.compressed_multiset,
+                                       prev.union_compressed_multiset))::numeric,
+            10)
  ORDER BY curr.recno;
 
 -- Test aggregate accumulation