serve challenges following insertion order (#240) (#241)

Author: zeegomo

Cargo.lock

@@ -152,7 +152,7 @@ paths:
       summary: Get all available challenges
       tags: [challenge]
       description: |
-        Lists all available challenges.
+        Lists all available challenges following insertion order.
       responses:
         "200":
           description: Valid response

doc/api/v0.yaml

@@ -16,6 +16,7 @@ rand = "0.7.3"
 r2d2 = "0.8"
 structopt = "0.3.14"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
 tempfile = "3.1.0"
 thiserror = "1.0"
 vit-servicing-station-lib = { path = "../vit-servicing-station-lib" }

vit-servicing-station-cli/Cargo.toml

@@ -2,7 +2,6 @@ use crate::db_utils::{backup_db_file, restore_db_file};
 use crate::{db_utils::db_file_exists, task::ExecTask};
 use csv::Trim;
 use serde::de::DeserializeOwned;
-use std::collections::HashMap;
 use std::convert::TryInto;
 use std::io;
 use std::path::{Path, PathBuf};
@@ -14,7 +13,7 @@ use vit_servicing_station_lib::db::models::proposals::{
 };
 use vit_servicing_station_lib::db::{
     load_db_connection_pool,
-    models::{challenges::Challenge, funds::Fund, proposals::Proposal, voteplans::Voteplan},
+    models::{funds::Fund, proposals::Proposal, voteplans::Voteplan},
 };
 
 #[derive(Error, Debug)]
@@ -107,11 +106,8 @@ impl CsvDataCmd {
         let funds = CsvDataCmd::load_from_csv::<Fund>(funds_path)?;
 
         let mut voteplans = CsvDataCmd::load_from_csv::<Voteplan>(voteplans_path)?;
-        let mut challenges: HashMap<i32, Challenge> =
-            CsvDataCmd::load_from_csv::<Challenge>(challenges_path)?
-                .into_iter()
-                .map(|c| (c.id, c))
-                .collect();
+        let mut challenges =
+            CsvDataCmd::load_from_csv::<super::models::Challenge>(challenges_path)?;
         let csv_proposals = CsvDataCmd::load_from_csv::<super::models::Proposal>(proposals_path)?;
         let reviews = CsvDataCmd::load_from_csv::<super::models::AdvisorReview>(reviews_path)?
             .into_iter()
@@ -125,7 +121,8 @@ impl CsvDataCmd {
 
         for proposal in csv_proposals {
             let challenge_type = challenges
-                .get(&proposal.challenge_id)
+                .iter()
+                .find(|c| proposal.challenge_id == c.id)
                 .ok_or_else(|| {
                     std::io::Error::new(
                         std::io::ErrorKind::InvalidData,
@@ -181,7 +178,7 @@ impl CsvDataCmd {
         }
 
         // apply fund id to challenges
-        for challenge in challenges.values_mut() {
+        for challenge in challenges.iter_mut() {
             challenge.fund_id = fund.id;
         }
 
@@ -212,7 +209,10 @@ impl CsvDataCmd {
         .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("{}", e)))?;
 
         vit_servicing_station_lib::db::queries::challenges::batch_insert_challenges(
-            &challenges.values().cloned().collect::<Vec<Challenge>>(),
+            &challenges
+                .into_iter()
+                .map(|c| c.into_db_challenge_values())
+                .collect::<Vec<_>>(),
             &db_conn,
         )
         .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("{}", e)))?;

vit-servicing-station-cli/src/csv/loaders.rs

@@ -1,10 +1,33 @@
+use diesel::{ExpressionMethods, Insertable};
 use serde::Deserialize;
 use std::convert::TryInto;
+use vit_servicing_station_lib::db::models::challenges::{
+    Challenge as DbChallenge, ChallengeHighlights,
+};
 use vit_servicing_station_lib::db::models::community_advisors_reviews::{self, ReviewRanking};
 use vit_servicing_station_lib::db::models::proposals::{
     self, community_choice, simple, Category, ChallengeType, ProposalChallengeInfo, Proposer,
 };
 use vit_servicing_station_lib::db::models::vote_options::VoteOptions;
+use vit_servicing_station_lib::db::schema::challenges;
+
+#[derive(Deserialize, Clone, Debug, PartialEq, Eq)]
+pub struct Challenge {
+    pub id: i32,
+    #[serde(alias = "challengeType")]
+    pub challenge_type: ChallengeType,
+    pub title: String,
+    pub description: String,
+    #[serde(alias = "rewardsTotal")]
+    pub rewards_total: i64,
+    #[serde(alias = "proposersRewards")]
+    pub proposers_rewards: i64,
+    #[serde(alias = "fundId")]
+    pub fund_id: i32,
+    #[serde(alias = "challengeUrl")]
+    pub challenge_url: String,
+    pub highlights: Option<ChallengeHighlights>,
+}
 
 #[derive(Deserialize, PartialEq, Eq, Debug, Clone)]
 pub struct Proposal {
@@ -104,6 +127,25 @@ fn default_challenge_id() -> i32 {
     -1
 }
 
+impl Challenge {
+    pub fn into_db_challenge_values(
+        self,
+    ) -> <DbChallenge as Insertable<challenges::table>>::Values {
+        (
+            challenges::id.eq(self.id),
+            challenges::challenge_type.eq(self.challenge_type.to_string()),
+            challenges::title.eq(self.title),
+            challenges::description.eq(self.description),
+            challenges::rewards_total.eq(self.rewards_total),
+            challenges::proposers_rewards.eq(self.proposers_rewards),
+            challenges::fund_id.eq(self.fund_id),
+            challenges::challenge_url.eq(self.challenge_url),
+            // This should always be a valid json
+            challenges::highlights.eq(serde_json::to_string(&self.highlights).ok()),
+        )
+    }
+}
+
 impl Proposal {
     pub fn into_db_proposal_and_challenge_info(
         self,

vit-servicing-station-cli/src/csv/models.rs

@@ -84,8 +84,9 @@ create table api_tokens
 
 create table challenges
 (
-    id INTEGER NOT NULL
+    internal_id INTEGER NOT NULL
         primary key autoincrement,
+    id INTEGER NOT NULL UNIQUE,
     challenge_type VARCHAR NOT NULL,
     title VARCHAR NOT NULL,
     description VARCHAR NOT NULL,

vit-servicing-station-lib/migrations/2020-05-22-112032_setup_db/up.sql

@@ -10,6 +10,9 @@ pub struct ChallengeHighlights {
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 pub struct Challenge {
+    #[serde(alias = "internalId")]
+    // this is used only to retain the original insert order
+    pub internal_id: i32,
     pub id: i32,
     #[serde(alias = "challengeType")]
     pub challenge_type: ChallengeType,
@@ -28,38 +31,41 @@ pub struct Challenge {
 
 impl Queryable<challenges::SqlType, Db> for Challenge {
     type Row = (
-        // 0 -> id
+        // 0 -> internal_id
         i32,
-        // 1 -> challenge_type
+        // 1 -> id
+        i32,
+        // 2 -> challenge_type
         String,
-        // 1 -> title
+        // 3 -> title
         String,
-        // 2 -> description
+        // 4 -> description
         String,
-        // 3 -> rewards_total
+        // 5 -> rewards_total
         i64,
-        // 4 -> proposers_rewards
+        // 6 -> proposers_rewards
         i64,
-        // 5 -> fund_id
+        // 7 -> fund_id
         i32,
-        // 6 -> fund_url
+        // 8 -> fund_url
         String,
-        // 7 -> challenge_highlights
+        // 9 -> challenge_highlights
         Option<String>,
     );
 
     fn build(row: Self::Row) -> Self {
         Challenge {
-            id: row.0,
-            challenge_type: row.1.parse().unwrap(),
-            title: row.2,
-            description: row.3,
-            rewards_total: row.4,
-            proposers_rewards: row.5,
-            fund_id: row.6,
-            challenge_url: row.7,
+            internal_id: row.0,
+            id: row.1,
+            challenge_type: row.2.parse().unwrap(),
+            title: row.3,
+            description: row.4,
+            rewards_total: row.5,
+            proposers_rewards: row.6,
+            fund_id: row.7,
+            challenge_url: row.8,
             // It should be ensured that the content is valid json
-            highlights: row.8.and_then(|v| serde_json::from_str(&v).ok()),
+            highlights: row.9.and_then(|v| serde_json::from_str(&v).ok()),
         }
     }
 }
@@ -104,6 +110,7 @@ pub mod test {
         const CHALLENGE_ID: i32 = 9001;
         const REWARDS_TOTAL: i64 = 100500;
         Challenge {
+            internal_id: 1,
             id: CHALLENGE_ID,
             challenge_type: ChallengeType::CommunityChoice,
             title: "challenge title".to_string(),

vit-servicing-station-lib/src/db/models/challenges.rs

@@ -7,14 +7,17 @@ use crate::{
     },
     v0::errors::HandleError,
 };
-use diesel::{ExpressionMethods, Insertable, QueryResult, RunQueryDsl};
+use diesel::{ExpressionMethods, Insertable, QueryDsl, QueryResult, RunQueryDsl};
 
 pub async fn query_all_challenges(pool: &DbConnectionPool) -> Result<Vec<Challenge>, HandleError> {
     let db_conn = pool.get().map_err(HandleError::DatabaseError)?;
     tokio::task::spawn_blocking(move || {
-        challenges_dsl::challenges
-            .load::<Challenge>(&db_conn)
-            .map_err(|_| HandleError::InternalError("Error retrieving challenges".to_string()))
+        diesel::QueryDsl::order_by(
+            challenges_dsl::challenges,
+            challenges::dsl::internal_id.asc(),
+        )
+        .load::<Challenge>(&db_conn)
+        .map_err(|_| HandleError::InternalError("Error retrieving challenges".to_string()))
     })
     .await
     .map_err(|_e| HandleError::InternalError("Error executing request".to_string()))?
@@ -44,6 +47,7 @@ pub async fn query_challenges_by_fund_id(
             challenges_dsl::challenges,
             challenges_dsl::fund_id.eq(fund_id),
         )
+        .order_by(challenges::dsl::internal_id.asc())
         .load::<Challenge>(&db_conn)
         .map_err(|_e| HandleError::NotFound("Error loading challenges for fund id".to_string()))
     })
@@ -69,16 +73,10 @@ pub async fn query_challenge_proposals_by_id(
 }
 
 pub fn batch_insert_challenges(
-    challenges_slice: &[Challenge],
+    challenges: &[<Challenge as Insertable<challenges::table>>::Values],
     db_conn: &DbConnection,
 ) -> QueryResult<usize> {
     diesel::insert_into(challenges::table)
-        .values(
-            challenges_slice
-                .iter()
-                .cloned()
-                .map(|challenge| challenge.values())
-                .collect::<Vec<_>>(),
-        )
+        .values(challenges)
         .execute(db_conn)
 }

vit-servicing-station-lib/src/db/queries/challenges.rs

@@ -8,6 +8,7 @@ table! {
 
 table! {
     challenges (id) {
+        internal_id -> Integer,
         id -> Integer,
         challenge_type -> Text,
         title -> Text,

vit-servicing-station-lib/src/db/schema.rs

@@ -266,6 +266,7 @@ impl ArbitrarySnapshotGenerator {
 
         vec![
             Challenge {
+                internal_id: first_challenge.internal_id,
                 id: simple_id.abs(),
                 challenge_type: ChallengeType::Simple,
                 title: first_challenge.title,
@@ -277,6 +278,7 @@ impl ArbitrarySnapshotGenerator {
                 highlights: self.template_generator.gen_highlights(),
             },
             Challenge {
+                internal_id: second_challenge.internal_id,
                 id: community_choice_id.abs(),
                 challenge_type: ChallengeType::CommunityChoice,
                 title: second_challenge.title,
@@ -295,6 +297,7 @@ impl ArbitrarySnapshotGenerator {
         let challenge = self.template_generator.next_challenge();
 
         Challenge {
+            internal_id: challenge.internal_id,
             id: id.abs(),
             challenge_type: ChallengeType::CommunityChoice,
             title: challenge.title,