Skip to content

Commit 75d8332

Browse files
djmbdjmb
committed
Adjust size estimate for encrypted entries
Add 170 bytes to the estimated size of each entry when encryption is enabled. This assumes that the default context properties are used.
1 parent 86fb61c commit 75d8332

File tree

6 files changed

+98
-69
lines changed

6 files changed

+98
-69
lines changed

‎Rakefile‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,7 @@ def run_without_aborting(*tasks)
2323
end
2424

2525
def configs
26-
<<<<<<< HEAD
27-
[ :default, :connects_to, :database, :no_database, :shards, :unprepared_statements ]
28-
=======
2926
[ :default, :connects_to, :database, :encrypted, :encrypted_custom, :no_database, :shards, :unprepared_statements ]
30-
>>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
3127
end
3228

3329
task :test do

‎app/models/solid_cache/entry.rb‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ class Entry < Record
88
# Based on experimentation on SQLite, MySQL and Postgresql.
99
# A bit high for SQLite (more like 90 bytes), but about right for MySQL/Postgresql.
1010
ESTIMATED_ROW_OVERHEAD = 140
11+
12+
# Assuming MessagePack serialization
13+
ESTIMATED_ENCRYPTION_OVERHEAD = 170
14+
1115
KEY_HASH_ID_RANGE = -(2**63)..(2**63 - 1)
1216

1317
class << self
@@ -149,7 +153,15 @@ def key_hashes_for(keys)
149153
end
150154

151155
def byte_size_for(payload)
152-
payload[:key].to_s.bytesize + payload[:value].to_s.bytesize + ESTIMATED_ROW_OVERHEAD
156+
payload[:key].to_s.bytesize + payload[:value].to_s.bytesize + estimated_row_overhead
157+
end
158+
159+
def estimated_row_overhead
160+
if SolidCache.configuration.encrypt?
161+
ESTIMATED_ROW_OVERHEAD + ESTIMATED_ENCRYPTION_OVERHEAD
162+
else
163+
ESTIMATED_ROW_OVERHEAD
164+
end
153165
end
154166
end
155167
end

‎test/models/solid_cache/entry/size/estimate_test.rb‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,32 @@
44

55
module SolidCache
66
class EntrySizeEstimateTest < ActiveSupport::TestCase
7+
setup do
8+
@encrypted = SolidCache.configuration.encrypt?
9+
end
10+
711
test "write and read cache entries" do
812
assert_equal 0, estimate(samples: 10)
913
end
1014

1115
test "gets exact estimate when samples sizes are big enough" do
1216
write_entries(value_lengths: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ])
1317

14-
assert_equal 1535, estimate(samples: 12)
15-
assert_equal 1535, estimate(samples: 10)
16-
assert_equal 1688, estimate(samples: 6)
17-
assert_equal 1689, estimate(samples: 5)
18+
assert_equal @encrypted ? 3235 : 1535, estimate(samples: 12)
19+
assert_equal @encrypted ? 3235 : 1535, estimate(samples: 10)
20+
assert_equal @encrypted ? 3558 : 1688, estimate(samples: 6)
21+
assert_equal @encrypted ? 3559 : 1689, estimate(samples: 5)
1822
end
1923

2024
test "test larger sample estimates" do
2125
values_lengths = with_fixed_srand(1) { 1000.times.map { (rand**2 * 1000).to_i } }
2226
write_entries(value_lengths: values_lengths)
2327

24-
assert_equal 481257, estimate(samples: 1000)
25-
assert_equal 481662, estimate(samples: 500)
26-
with_fixed_srand(1) { assert_equal 501624, estimate(samples: 100) }
27-
with_fixed_srand(1) { assert_equal 477621, estimate(samples: 50) }
28-
with_fixed_srand(1) { assert_equal 471878, estimate(samples: 10) }
28+
assert_equal @encrypted ? 651257 : 481257, estimate(samples: 1000)
29+
assert_equal @encrypted ? 651832 : 481662, estimate(samples: 500)
30+
with_fixed_srand(1) { assert_equal @encrypted ? 680804 : 501624, estimate(samples: 100) }
31+
with_fixed_srand(1) { assert_equal @encrypted ? 660541 : 477621, estimate(samples: 50) }
32+
with_fixed_srand(1) { assert_equal @encrypted ? 692368 : 471878, estimate(samples: 10) }
2933
end
3034

3135
test "test with gaps in records estimates" do
@@ -34,12 +38,12 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
3438
first_mod = Entry.first.id % 3
3539
Entry.where("id % 3 = #{first_mod}").delete_all
3640

37-
assert_equal 324532, estimate(samples: 1000)
38-
assert_equal 324741, estimate(samples: 500)
39-
with_fixed_srand(1) { assert_equal 323946, estimate(samples: 334) }
40-
with_fixed_srand(1) { assert_equal 345103, estimate(samples: 100) }
41-
with_fixed_srand(1) { assert_equal 335770, estimate(samples: 50) }
42-
with_fixed_srand(1) { assert_equal 281944, estimate(samples: 10) }
41+
assert_equal @encrypted ? 437752 : 324532, estimate(samples: 1000)
42+
assert_equal @encrypted ? 438131 : 324741, estimate(samples: 500)
43+
with_fixed_srand(1) { assert_equal @encrypted ? 437166 : 323946, estimate(samples: 334) }
44+
with_fixed_srand(1) { assert_equal @encrypted ? 462859 : 345103, estimate(samples: 100) }
45+
with_fixed_srand(1) { assert_equal @encrypted ? 453859 : 335770, estimate(samples: 50) }
46+
with_fixed_srand(1) { assert_equal @encrypted ? 401216 : 281944, estimate(samples: 10) }
4347
end
4448

4549
test "test with more gaps in records estimates" do
@@ -48,12 +52,12 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
4852
first_mod = Entry.first.id % 4
4953
Entry.where("id % 4 != #{first_mod}").delete_all
5054

51-
assert_equal 120304, estimate(samples: 1000)
52-
assert_equal 121488, estimate(samples: 500)
53-
with_fixed_srand(1) { assert_equal 121188, estimate(samples: 250) }
54-
with_fixed_srand(1) { assert_equal 126768, estimate(samples: 100) }
55-
with_fixed_srand(1) { assert_equal 132657, estimate(samples: 50) }
56-
with_fixed_srand(1) { assert_equal 25537, estimate(samples: 10) }
55+
assert_equal @encrypted ? 162804 : 120304, estimate(samples: 1000)
56+
assert_equal @encrypted ? 165348 : 121488, estimate(samples: 500)
57+
with_fixed_srand(1) { assert_equal @encrypted ? 164704 : 121188, estimate(samples: 250) }
58+
with_fixed_srand(1) { assert_equal @encrypted ? 174266 : 126768, estimate(samples: 100) }
59+
with_fixed_srand(1) { assert_equal @encrypted ? 179794 : 132657, estimate(samples: 50) }
60+
with_fixed_srand(1) { assert_equal @encrypted ? 44016 : 25537, estimate(samples: 10) }
5761
end
5862

5963
test "overestimate when all samples sizes are the same" do
@@ -62,11 +66,11 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
6266
# estimate in this case.
6367
write_entries(value_lengths: [1] * 1000)
6468

65-
assert_equal 149000, estimate(samples: 1000)
66-
assert_equal 297851, estimate(samples: 999)
67-
assert_equal 223500, estimate(samples: 500)
68-
with_fixed_srand(1) { assert_equal 272422, estimate(samples: 6) }
69-
with_fixed_srand(1) { assert_equal 326906, estimate(samples: 5) }
69+
assert_equal @encrypted ? 319000 : 149000, estimate(samples: 1000)
70+
assert_equal @encrypted ? 637681 : 297851, estimate(samples: 999)
71+
assert_equal @encrypted ? 478500 : 223500, estimate(samples: 500)
72+
with_fixed_srand(1) { assert_equal @encrypted ? 583238 : 272422, estimate(samples: 6) }
73+
with_fixed_srand(1) { assert_equal @encrypted ? 699886 : 326906, estimate(samples: 5) }
7074
end
7175

7276
private

‎test/models/solid_cache/entry/size/moving_average_estimate_test.rb‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44

55
module SolidCache
66
class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
7+
setup do
8+
@encrypted = SolidCache.configuration.encrypt?
9+
end
10+
711
test "write and read cache entries" do
812
assert_equal 0, estimate(samples: 10)
913
end
@@ -13,7 +17,7 @@ class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
1317

1418
estimate = Entry::Size::MovingAverageEstimate.new(samples: 12)
1519
assert_predicate estimate, :exact?
16-
assert_equal 1535, estimate.size
20+
assert_equal @encrypted ? 3235 : 1535, estimate.size
1721
end
1822

1923
test "tracks moving average" do
@@ -22,24 +26,24 @@ class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
2226
Entry.write Entry::Size::MovingAverageEstimate::ESTIMATES_KEY, "4637774|4754378|7588547"
2327

2428
with_fixed_srand(1) do
25-
assert_equal 10449357, estimate(samples: 1)
29+
assert_equal @encrypted ? 11016081 : 10449357, estimate(samples: 1)
2630
end
2731

28-
assert_equal "4754378|7588547|19005147", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
32+
assert_equal @encrypted ? "4754378|7588547|20705317" : "4754378|7588547|19005147", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
2933
end
3034

3135
test "appends to moving average when less than required items" do
3236
write_entries(value_lengths: 5000.times.to_a)
3337

3438
assert_nil Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
3539

36-
with_fixed_srand(1) { assert_equal 20991897, estimate(samples: 2) }
40+
with_fixed_srand(1) { assert_equal @encrypted ? 22691557 : 20991897, estimate(samples: 2) }
3741

38-
assert_equal "20991897", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
42+
assert_equal @encrypted ? "22691557" : "20991897", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
3943

40-
with_fixed_srand(2) { assert_equal 11917062, estimate(samples: 2) }
44+
with_fixed_srand(2) { assert_equal @encrypted ? 13191977 : 11917062, estimate(samples: 2) }
4145

42-
assert_equal "20991897|2842227", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
46+
assert_equal @encrypted ? "22691557|3692397" : "20991897|2842227", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
4347
end
4448

4549
private

‎test/models/solid_cache/entry/size_test.rb‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,31 @@
44

55
module SolidCache
66
class EntrySizeTest < ActiveSupport::TestCase
7+
setup do
8+
@encrypted = SolidCache.configuration.encrypt?
9+
end
10+
711
test "write and read cache entries" do
812
assert_equal 0, Entry.estimated_size
913
end
1014

1115
test "gets exact estimate when samples sizes are big enough" do
1216
write_entries(value_lengths: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ])
1317

14-
assert_equal 1535, Entry.estimated_size(samples: 12)
15-
assert_equal 1878, Entry.estimated_size(samples: 10)
16-
assert_equal 1882, Entry.estimated_size(samples: 6)
18+
assert_equal @encrypted ? 3235 : 1535, Entry.estimated_size(samples: 12)
19+
assert_equal @encrypted ? 3918 : 1878, Entry.estimated_size(samples: 10)
20+
assert_equal @encrypted ? 3922 : 1882, Entry.estimated_size(samples: 6)
1721
end
1822

1923
test "test larger sample estimates" do
2024
values_lengths = with_fixed_srand(1) { 1000.times.map { (rand**2 * 1000).to_i } }
2125
write_entries(value_lengths: values_lengths)
2226

23-
assert_equal 481257, Entry.estimated_size(samples: 1000)
24-
assert_equal 482262, Entry.estimated_size(samples: 501)
25-
with_fixed_srand(1) { assert_equal 502065, Entry.estimated_size(samples: 100) }
26-
with_fixed_srand(1) { assert_equal 478066, Entry.estimated_size(samples: 50) }
27-
with_fixed_srand(1) { assert_equal 472343, Entry.estimated_size(samples: 10) }
27+
assert_equal @encrypted ? 651257 : 481257, Entry.estimated_size(samples: 1000)
28+
assert_equal @encrypted ? 652772 : 482262, Entry.estimated_size(samples: 501)
29+
with_fixed_srand(1) { assert_equal @encrypted ? 681425 : 502065, Entry.estimated_size(samples: 100) }
30+
with_fixed_srand(1) { assert_equal @encrypted ? 661170 : 478066, Entry.estimated_size(samples: 50) }
31+
with_fixed_srand(1) { assert_equal @encrypted ? 693054 : 472343, Entry.estimated_size(samples: 10) }
2832
end
2933

3034
test "test with gaps in records estimates" do
@@ -33,12 +37,12 @@ class EntrySizeTest < ActiveSupport::TestCase
3337
first_mod = Entry.first.id % 3
3438
Entry.where("id % 3 = #{first_mod}").delete_all
3539

36-
assert_equal 324532, Entry.estimated_size(samples: 1000)
37-
assert_equal 324936, Entry.estimated_size(samples: 500)
38-
with_fixed_srand(1) { assert_equal 324567, Entry.estimated_size(samples: 334) }
39-
with_fixed_srand(1) { assert_equal 345649, Entry.estimated_size(samples: 100) }
40-
with_fixed_srand(1) { assert_equal 336366, Entry.estimated_size(samples: 50) }
41-
with_fixed_srand(1) { assert_equal 282492, Entry.estimated_size(samples: 10) }
40+
assert_equal @encrypted ? 437752 : 324532, Entry.estimated_size(samples: 1000)
41+
assert_equal @encrypted ? 438496 : 324936, Entry.estimated_size(samples: 500)
42+
with_fixed_srand(1) { assert_equal @encrypted ? 438296 : 324567, Entry.estimated_size(samples: 334) }
43+
with_fixed_srand(1) { assert_equal @encrypted ? 463629 : 345649, Entry.estimated_size(samples: 100) }
44+
with_fixed_srand(1) { assert_equal @encrypted ? 454686 : 336366, Entry.estimated_size(samples: 50) }
45+
with_fixed_srand(1) { assert_equal @encrypted ? 402002 : 282492, Entry.estimated_size(samples: 10) }
4246
end
4347

4448
test "test with more gaps in records estimates" do
@@ -47,12 +51,12 @@ class EntrySizeTest < ActiveSupport::TestCase
4751
first_mod = Entry.first.id % 4
4852
Entry.where("id % 4 != #{first_mod}").delete_all
4953

50-
assert_equal 120304, Entry.estimated_size(samples: 1000)
51-
assert_equal 121683, Entry.estimated_size(samples: 501)
52-
with_fixed_srand(1) { assert_equal 121240, Entry.estimated_size(samples: 250) }
53-
with_fixed_srand(1) { assert_equal 126976, Entry.estimated_size(samples: 100) }
54-
with_fixed_srand(1) { assert_equal 133014, Entry.estimated_size(samples: 50) }
55-
with_fixed_srand(1) { assert_equal 25596, Entry.estimated_size(samples: 10) }
54+
assert_equal @encrypted ? 162804 : 120304, Entry.estimated_size(samples: 1000)
55+
assert_equal @encrypted ? 165713 : 121683, Entry.estimated_size(samples: 501)
56+
with_fixed_srand(1) { assert_equal @encrypted ? 164762 : 121240, Entry.estimated_size(samples: 250) }
57+
with_fixed_srand(1) { assert_equal @encrypted ? 174610 : 126976, Entry.estimated_size(samples: 100) }
58+
with_fixed_srand(1) { assert_equal @encrypted ? 180315 : 133014, Entry.estimated_size(samples: 50) }
59+
with_fixed_srand(1) { assert_equal @encrypted ? 44143 : 25596, Entry.estimated_size(samples: 10) }
5660
end
5761

5862
test "overestimate when all samples sizes are the same" do
@@ -61,11 +65,11 @@ class EntrySizeTest < ActiveSupport::TestCase
6165
# estimate in this case.
6266
write_entries(value_lengths: [1] * 1000)
6367

64-
assert_equal 149000, Entry.estimated_size(samples: 1000)
65-
assert_equal 297897, Entry.estimated_size(samples: 999)
66-
assert_equal 223695, Entry.estimated_size(samples: 501)
67-
with_fixed_srand(1) { assert_equal 272741, Entry.estimated_size(samples: 6) }
68-
with_fixed_srand(1) { assert_equal 327280, Entry.estimated_size(samples: 5) }
68+
assert_equal @encrypted ? 319000 : 149000, Entry.estimated_size(samples: 1000)
69+
assert_equal @encrypted ? 637727 : 297897, Entry.estimated_size(samples: 999)
70+
assert_equal @encrypted ? 478865 : 223695, Entry.estimated_size(samples: 501)
71+
with_fixed_srand(1) { assert_equal @encrypted ? 583869 : 272741, Entry.estimated_size(samples: 6) }
72+
with_fixed_srand(1) { assert_equal @encrypted ? 700634 : 327280, Entry.estimated_size(samples: 5) }
6973
end
7074

7175
private

‎test/models/solid_cache/entry_test.rb‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,21 @@ class EntryTest < ActiveSupport::TestCase
5353
end
5454

5555
test "byte_size" do
56-
Entry.write "hello".b, "test"
57-
assert_equal 149, Entry.uncached { Entry.last.byte_size }
58-
Entry.write "hello".b, "12345"
59-
assert_equal 150, Entry.uncached { Entry.last.byte_size }
60-
Entry.write "hi".b, "12345"
61-
assert_equal 147, Entry.uncached { Entry.last.byte_size }
56+
if SolidCache.configuration.encrypt?
57+
Entry.write "hello".b, "test"
58+
assert_equal 319, Entry.uncached { Entry.last.byte_size }
59+
Entry.write "hello".b, "12345"
60+
assert_equal 320, Entry.uncached { Entry.last.byte_size }
61+
Entry.write "hi".b, "12345"
62+
assert_equal 317, Entry.uncached { Entry.last.byte_size }
63+
else
64+
Entry.write "hello".b, "test"
65+
assert_equal 149, Entry.uncached { Entry.last.byte_size }
66+
Entry.write "hello".b, "12345"
67+
assert_equal 150, Entry.uncached { Entry.last.byte_size }
68+
Entry.write "hi".b, "12345"
69+
assert_equal 147, Entry.uncached { Entry.last.byte_size }
70+
end
6271
end
6372

6473
private

0 commit comments

Comments
 (0)