Implement --minimum-confidence parameter for vCA reward command

catalyst-toolbox/src/bin/cli/rewards/veterans.rs

@@ -62,6 +62,14 @@ pub struct VeteransRewards {
     /// if the first cutoff is selected then the first modifier is used.
     #[structopt(long, required = true)]
     reputation_agreement_rate_modifiers: Vec<Decimal>,
+
+    /// Value in range [0.5, 1]
+    /// The minimum consensus for a vCA ranking to be excluded from eligible rankings when in disagreement from simple majority.
+    /// Simple majority is 50%.
+    /// Qualified majority is 70%. Using 70% avoids punishing vCAs where the consensus is not clear.
+    /// 70% is because when #vca == 3 consensus is only 66% and thus in this case, where there is just 1 vote in disagreement, all 3 vCAs get rewarded.
+    #[structopt(long)]
+    minimum_consensus: Decimal,
 }
 
 impl VeteransRewards {
@@ -77,6 +85,7 @@ impl VeteransRewards {
             rewards_agreement_rate_modifiers,
             reputation_agreement_rate_cutoffs,
             reputation_agreement_rate_modifiers,
+            minimum_consensus,
         } = self;
         let reviews: Vec<VeteranRankingRow> = csv::load_data_from_csv::<_, b','>(&from)?;
 
@@ -100,6 +109,10 @@ impl VeteransRewards {
             bail!("Expected rewards_agreement_rate_cutoffs to be descending");
         }
 
+        if minimum_consensus < Decimal::new(5,1) || minimum_consensus > Decimal::ONE {
+            bail!("Expected minimum_consensus to range between .5 and 1");
+        }
+
         let results = veterans::calculate_veteran_advisors_incentives(
             &reviews,
             total_rewards,
@@ -113,6 +126,7 @@ impl VeteransRewards {
                 .into_iter()
                 .zip(reputation_agreement_rate_modifiers.into_iter())
                 .collect(),
+            minimum_consensus,
         );
 
         csv::dump_data_to_csv(rewards_to_csv_data(results).iter(), &to).unwrap();

catalyst-toolbox/src/rewards/veterans.rs

@@ -16,6 +16,11 @@ pub struct VeteranAdvisorIncentive {
     pub reputation: u64,
 }
 
+pub struct FinalRankingWithConsensus {
+    pub review_ranking: ReviewRanking,
+    pub consensus: Decimal,
+}
+
 pub type VcaRewards = HashMap<VeteranAdvisorId, VeteranAdvisorIncentive>;
 pub type EligibilityThresholds = std::ops::RangeInclusive<usize>;
 
@@ -25,18 +30,34 @@ pub type EligibilityThresholds = std::ops::RangeInclusive<usize>;
 // e.g. something like an expanded version of a AdvisorReviewRow
 // [proposal_id, advisor, ratings, ..(other fields from AdvisorReviewRow).., ranking (good/excellent/filtered out), vca]
 
-fn calc_final_ranking_per_review(rankings: &[impl Borrow<VeteranRankingRow>]) -> ReviewRanking {
+// note that the consensus returned here is either FO/total or ((E+G)/total) because for now we do
+// not discriminate between Excellent and Good...
+fn calc_final_ranking_with_consensus_per_review(rankings: &[impl Borrow<VeteranRankingRow>]) -> FinalRankingWithConsensus {
     let rankings_majority = Decimal::from(rankings.len()) / Decimal::from(2);
     let ranks = rankings.iter().counts_by(|r| r.borrow().score());
 
-    match (ranks.get(&FilteredOut), ranks.get(&Excellent)) {
-        (Some(filtered_out), _) if Decimal::from(*filtered_out) >= rankings_majority => {
-            ReviewRanking::FilteredOut
+    match (ranks.get(&Excellent), ranks.get(&Good), ranks.get(&FilteredOut)) {
+        (_, _, Some(filtered_out)) if Decimal::from(*filtered_out) >= rankings_majority => {
+            FinalRankingWithConsensus {
+                review_ranking: FilteredOut,
+                consensus: Decimal::from(*filtered_out) / Decimal::from(rankings.len()),
+            }
         }
-        (_, Some(excellent)) if Decimal::from(*excellent) > rankings_majority => {
-            ReviewRanking::Excellent
+        (Some(excellent), maybe_good, _) if Decimal::from(*excellent) > rankings_majority => {
+            FinalRankingWithConsensus {
+                review_ranking: Excellent,
+                consensus: (maybe_good.map(|good| Decimal::from(*good)).unwrap_or_default()
+                    + Decimal::from(*excellent)) / Decimal::from(rankings.len()),
+            }
         }
-        _ => ReviewRanking::Good,
+        (maybe_excellent, Some(good), _) => {
+            FinalRankingWithConsensus {
+                review_ranking: Good,
+                consensus: (maybe_excellent.map(|excellent| Decimal::from(*excellent)).unwrap_or_default()
+                    + Decimal::from(*good)) / Decimal::from(rankings.len()),
+            }
+        }
+        _ => unreachable!(),
     }
 }
 
@@ -77,19 +98,21 @@ fn calc_final_eligible_rankings(
         .collect()
 }
 
+
 pub fn calculate_veteran_advisors_incentives(
     veteran_rankings: &[VeteranRankingRow],
     total_rewards: Rewards,
     rewards_thresholds: EligibilityThresholds,
     reputation_thresholds: EligibilityThresholds,
     rewards_mod_args: Vec<(Decimal, Decimal)>,
     reputation_mod_args: Vec<(Decimal, Decimal)>,
+    minimum_consensus: Decimal,
 ) -> HashMap<VeteranAdvisorId, VeteranAdvisorIncentive> {
-    let final_rankings_per_review = veteran_rankings
+    let final_rankings_with_consensus_per_review = veteran_rankings
         .iter()
         .into_group_map_by(|ranking| ranking.review_id())
         .into_iter()
-        .map(|(review, rankings)| (review, calc_final_ranking_per_review(&rankings)))
+        .map(|(review, rankings)| (review, calc_final_ranking_with_consensus_per_review(&rankings)))
         .collect::<BTreeMap<_, _>>();
 
     let rankings_per_vca = veteran_rankings
@@ -99,11 +122,13 @@ pub fn calculate_veteran_advisors_incentives(
     let eligible_rankings_per_vca = veteran_rankings
         .iter()
         .filter(|ranking| {
-            final_rankings_per_review
+            let final_ranking_with_consensus = final_rankings_with_consensus_per_review
                 .get(&ranking.review_id())
-                .unwrap()
-                .is_positive()
-                == ranking.score().is_positive()
+                .unwrap();
+
+            final_ranking_with_consensus.review_ranking
+                .is_positive() == ranking.score().is_positive()
+                || final_ranking_with_consensus.consensus < minimum_consensus
         })
         .counts_by(|ranking| ranking.vca.clone());
 
@@ -156,6 +181,8 @@ mod tests {
     const VCA_1: &str = "vca1";
     const VCA_2: &str = "vca2";
     const VCA_3: &str = "vca3";
+    const SIMPLE_MINIMUM_CONSENSUS: Decimal = dec!(.5);
+    const QUALIFIED_MINIMUM_CONSENSUS: Decimal = dec!(.7);
 
     struct RandomIterator;
     impl Iterator for RandomIterator {
@@ -188,23 +215,39 @@ mod tests {
     #[test]
     fn final_ranking_is_correct() {
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 5, 5, 5, RandomIterator),),
-            ReviewRanking::Good
+            calc_final_ranking_with_consensus_per_review(&gen_dummy_rankings("".into(), 5, 5, 5, RandomIterator)),
+            FinalRankingWithConsensus {
+                review_ranking: Good,
+                consensus
+            } if consensus == (dec!(10) / dec!(15))
         ));
 
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 4, 2, 5, RandomIterator)),
-            ReviewRanking::Good
+            calc_final_ranking_with_consensus_per_review(&gen_dummy_rankings("".into(), 4, 2, 5, RandomIterator)),
+            FinalRankingWithConsensus {
+                review_ranking: Good,
+                consensus
+            } if consensus == (dec!(6) / dec!(11))
+
         ));
 
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 4, 1, 5, RandomIterator)),
-            ReviewRanking::FilteredOut
+            calc_final_ranking_with_consensus_per_review(&gen_dummy_rankings("".into(), 4, 1, 5, RandomIterator)),
+            FinalRankingWithConsensus {
+                review_ranking: FilteredOut,
+                consensus,
+            } if consensus == (dec!(5) / dec!(10))
         ));
 
+        let excellent_final =calc_final_ranking_with_consensus_per_review(&gen_dummy_rankings("".into(), 3, 1, 1, RandomIterator));
+
+            dbg!(excellent_final.consensus);
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 3, 1, 1, RandomIterator)),
-            ReviewRanking::Excellent
+            calc_final_ranking_with_consensus_per_review(&gen_dummy_rankings("".into(), 3, 1, 1, RandomIterator)),
+            FinalRankingWithConsensus {
+                review_ranking: Excellent,
+                consensus,
+            } if consensus == (dec!(4) / dec!(5))
         ));
     }
 
@@ -231,6 +274,7 @@ mod tests {
                 .into_iter()
                 .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
                 .collect(),
+            SIMPLE_MINIMUM_CONSENSUS,
         );
         assert!(results.get(VCA_1).is_none());
         let res = results.get(VCA_2).unwrap();
@@ -260,6 +304,7 @@ mod tests {
                 .into_iter()
                 .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
                 .collect(),
+            SIMPLE_MINIMUM_CONSENSUS,
         );
         let res1 = results.get(VCA_1).unwrap();
         assert_eq!(res1.reputation, 1);
@@ -283,12 +328,12 @@ mod tests {
             (Rewards::new(8, 1), Rewards::ONE, Rewards::ONE),
             (Rewards::new(9, 1), Rewards::new(125, 2), Rewards::ONE),
         ];
-        for (agreement, reward_modifier, reputation_modifier) in inputs {
+        for (vca3_agreement, reward_modifier, reputation_modifier) in inputs {
             let rankings = (0..100)
                 .flat_map(|i| {
                     let vcas =
                         vec![VCA_1.to_owned(), VCA_2.to_owned(), VCA_3.to_owned()].into_iter();
-                    let (good, filtered_out) = if Rewards::from(i) < agreement * Rewards::from(100)
+                    let (good, filtered_out) = if Rewards::from(i) < vca3_agreement * Rewards::from(100)
                     {
                         (3, 0)
                     } else {
@@ -297,7 +342,38 @@ mod tests {
                     gen_dummy_rankings(i.to_string(), 0, good, filtered_out, vcas).into_iter()
                 })
                 .collect::<Vec<_>>();
-            let results = calculate_veteran_advisors_incentives(
+            let results_simple_consensus = calculate_veteran_advisors_incentives(
+                &rankings,
+                total_rewards,
+                1..=200,
+                1..=200,
+                THRESHOLDS
+                    .into_iter()
+                    .zip(REWARDS_DISAGREEMENT_MODIFIERS.into_iter())
+                    .collect(),
+                THRESHOLDS
+                    .into_iter()
+                    .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
+                    .collect(),
+                SIMPLE_MINIMUM_CONSENSUS,
+            );
+            let vca3_expected_reward_portion_simple_consensus = vca3_agreement * Rewards::from(100) * reward_modifier;
+            dbg!(vca3_expected_reward_portion_simple_consensus);
+            dbg!(vca3_agreement, reward_modifier, reputation_modifier);
+            let vca3_expected_rewards_simple_consensus = total_rewards
+                / (Rewards::from(125 * 2) + vca3_expected_reward_portion_simple_consensus)
+                * vca3_expected_reward_portion_simple_consensus;
+            let res_vca3_simple_consensus = results_simple_consensus.get(VCA_3).unwrap();
+            assert_eq!(
+                res_vca3_simple_consensus.reputation,
+                (Rewards::from(100) * vca3_agreement * reputation_modifier)
+                    .to_u64()
+                    .unwrap()
+            );
+            assert!(are_close(res_vca3_simple_consensus.rewards, vca3_expected_rewards_simple_consensus));
+
+
+            let results_qualified_consensus = calculate_veteran_advisors_incentives(
                 &rankings,
                 total_rewards,
                 1..=200,
@@ -310,21 +386,27 @@ mod tests {
                     .into_iter()
                     .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
                     .collect(),
+                QUALIFIED_MINIMUM_CONSENSUS,
             );
-            let expected_reward_portion = agreement * Rewards::from(100) * reward_modifier;
-            dbg!(expected_reward_portion);
-            dbg!(agreement, reward_modifier, reputation_modifier);
-            let expected_rewards = total_rewards
-                / (Rewards::from(125 * 2) + expected_reward_portion)
-                * expected_reward_portion;
-            let res = results.get(VCA_3).unwrap();
+
+            let vca3_expected_reward_portion_qualified_consensus = Rewards::from(100) * dec!(1.25); // low consensus so max reward modifier, agreement ratio doesn't count as all and rankings are all eligible
+            dbg!(vca3_expected_reward_portion_qualified_consensus);
+            dbg!(vca3_agreement, reward_modifier, reputation_modifier);
+
+            let vca3_expected_rewards_qualified_consensus = total_rewards
+                / (Rewards::from(125 * 2) + vca3_expected_reward_portion_qualified_consensus)
+                * vca3_expected_reward_portion_qualified_consensus; // 1/3 of the reward
+
+            let res_vca3_qualified_consensus = results_qualified_consensus.get(VCA_3).unwrap();
+
+
             assert_eq!(
-                res.reputation,
-                (Rewards::from(100) * agreement * reputation_modifier)
+                res_vca3_qualified_consensus.reputation,
+                (Rewards::from(100)) // all assessment are valid since consensus is low (2/3 < 0.7)
                     .to_u64()
                     .unwrap()
             );
-            assert!(are_close(res.rewards, expected_rewards));
+            assert!(are_close(res_vca3_qualified_consensus.rewards, vca3_expected_rewards_qualified_consensus));
         }
     }
 }