Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mongodb #13

Open
wants to merge 21 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
import _root_.sbt.Keys._

name := "lyrics-engine"

version := "1.0-SNAPSHOT"

name := "lyrics-engine"

lazy val commonSettings = Seq(
Expand All @@ -16,10 +22,17 @@ lazy val parser = (project in file("parser"))
.dependsOn(domain)
lazy val api = (project in file("api"))
.settings(commonSettings: _*)

lazy val persistence = (project in file("persistence"))
.settings(commonSettings: _*)
.dependsOn(domain)

lazy val root = (project in file("."))
.settings(commonSettings: _*)
.dependsOn(api)
.aggregate(domain, parser, api)
.aggregate(domain, parser, api, persistence)
.enablePlugins(UniversalPlugin, JavaAppPackaging)




24 changes: 24 additions & 0 deletions persistence/build.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import sbt.Keys._

name := "persistence"

organization := "scalalab3.lyricsengine"

version := "0.1.0-SNAPSHOT"

scalaVersion := "2.11.8"

resolvers += "releases" at "https://oss.sonatype.org/content/groups/scala-tools"

val specsV = "3.7.2"
val scalaTestV = "2.2.6"

libraryDependencies ++=
Seq(
"org.scalatest" %% "scalatest" % scalaTestV,
"org.specs2" %% "specs2-core" % specsV,
"org.specs2" %% "specs2-matcher-extra" % specsV,
"org.mongodb" %% "casbah" % "3.1.1",
"com.typesafe" % "config" % "1.3.0"
)

10 changes: 10 additions & 0 deletions persistence/src/main/resources/application.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
mongo {
host= "localhost"
port= "8080"
user=""
password=""
dbname= "lyrics"
songscollection= "songs"
wordsdefinitions= "wordsdefinitions"
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package scalalab3.lyricsengine.persistence

import scalalab3.lyricsengine.domain._

/**
* Created by annie on 5/17/16.
*/
trait StorageComponent {

val storage: Storage

trait Storage {
def addDataSet(dataSet: DataSet, version: Option[Int] = None)

def getDataSet(version: Option[Int] = None): DataSet

def getLastVersion: Int
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package scalalab3.lyricsengine.persistence

import com.mongodb.casbah.Imports._

import scalalab3.lyricsengine.domain._
import scalalab3.lyricsengine.persistence.mongo.MongoContext

trait StorageComponentImpl extends StorageComponent {

override val storage: Storage

class StorageImpl(implicit context: MongoContext) extends Storage {

def addDataSet(dataSet: DataSet, version: Option[Int] = None) = {
addWordsDefinition(dataSet.definition, version)
addSongs(dataSet.songs, version)
}

def getDataSet(version: Option[Int] = None): DataSet = {
val wd = findWordsDefinitions(version)
val songs = findSongs(version)
DataSet(wd, songs)
}

def getLastVersion: Int = {
val query = MongoDBObject() // All documents
val fields = MongoDBObject("version" -> 1) // Only return `version`
val orderBy = MongoDBObject("version" -> -1) // Order by version descending
val result = context.wdCollection.findOne(query, fields, orderBy)
val defaultVersion = -1
result match {
case Some(res) => res.getAs[Int]("version").getOrElse(defaultVersion)
case None => defaultVersion
}
}

private def addSongs(songsToAdd: Seq[Song], version: Option[Int] = None) = {
val builder = context.songsCollection.initializeOrderedBulkOperation //will automatically split the operation into batches
for {
song <- songsToAdd
} builder.insert(songToMongoDBObj(song, version))
val result = builder.execute()
}

private def addWordsDefinition(wd: WordsDefinition, version: Option[Int] = None) = {
val builder = context.wdCollection.initializeOrderedBulkOperation
builder.insert(wdToMongoDbObject(wd, version))
val result = builder.execute()
}

private def findSongs(version: Option[Int] = None): Seq[Song] = {
val query = MongoDBObject("version" -> version.getOrElse(getLastVersion))
val result = context.songsCollection.find(query)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Вообще здесь можно обойтись без паттерн-матчинга. version.getOrElse(null) вполне хватит.

val songsSet = for {
song <- result
} yield songFromMongoDBObj(song)
songsSet.toSeq
}

private def findWordsDefinitions(version: Option[Int] = None): WordsDefinition = {
val query = MongoDBObject("version" -> version.getOrElse(getLastVersion))
val result = context.wdCollection.find(query)
result.map(wdFromMongoDBObj(_)).next()
}

private def countSongs(): Int = context.songsCollection.find().count()

private def countWD(): Int = context.wdCollection.find().count()

private def songFromMongoDBObj(obj: MongoDBObject): Song = {
val _msdTrackId = obj.as[String]("msdTrackId")
val _mxmTrackId = obj.as[String]("mxmTrackId")
val _words = obj.as[Map[Int, Int]]("words")
Song(msdTrackId = _msdTrackId, mxmTrackId = _mxmTrackId, words = _words)
}

private def wdFromMongoDBObj(obj: MongoDBObject): Map[Int, String] = {
obj.getAs[Map[String, String]]("wordsDefinitions").get.map { case (k, v) => (k.toInt, v.toString) }
}

private def songToMongoDBObj(song: Song, version: Option[Int]): MongoDBObject = {
val songBuilder = MongoDBObject.newBuilder
songBuilder +=("msdTrackId" -> song.msdTrackId,
"mxmTrackId" -> song.mxmTrackId,
"words" -> getWDefinitions(song),
"version" -> version.getOrElse(getLastVersion)
)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

songBuilder.result()
}

private def wdToMongoDbObject(wd: WordsDefinition, version: Option[Int]): MongoDBObject = {
val wdBuilder = MongoDBObject.newBuilder
wdBuilder +=(
"wordsDefinitions" -> transformWordsDef(wd),
"version" -> version.getOrElse(getLastVersion)
)
wdBuilder.result()
}

private def transformWordsDef(w: WordsDefinition): MongoDBObject = {
val collection = MongoDBObject.newBuilder
val transformed = w.map { case (k, v) => (k.toString, v) }
collection ++= transformed
collection.result()
}

private def getWDefinitions(song: Song): MongoDBObject = {
val words = MongoDBObject.newBuilder
val songWords = song.words.map { case (k, v) => (k.toString, v) }
words ++= songWords
words.result()
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package scalalab3.lyricsengine.persistence.mongo

import com.typesafe.config.ConfigFactory

import constants.DefaultConfigValues._
import constants.DefaultConfigKeys._
import scala.util.Try

case class MongoConfig(host: String = defaultHost,
port: Int = defaultPort,
user: String = defaultUser,
password: String = defaultPassword,
dbName: String = defaultDbName,
songsCollection: String = defaultSongsCollection,
wdCollection: String = defaultWDCollection)

object MongoConfig {
private val config = ConfigFactory.load()

def load(): MongoConfig =
MongoConfig(
getString(host, defaultHost),
getInt(port, defaultPort),
getString(user, defaultUser),
getString(password, defaultPassword),
getString(dbname, defaultDbName),
getString(songsCollection, defaultSongsCollection),
getString(wordsDefinitions, defaultWDCollection)
)
private def getString(key: String, defaultValue: String) = Try(config.getString(key)).getOrElse(defaultValue)
private def getInt(key: String, defaultValue: Int) = Try(config.getInt(key)).getOrElse(defaultValue)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package scalalab3.lyricsengine.persistence.mongo

import com.mongodb.BasicDBObjectBuilder
import com.mongodb.casbah.Imports._
import com.mongodb.casbah.MongoClient

class MongoContext(val config: MongoConfig) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

С конфигами вы очень намудрили.
Не понял смысла выносить отдельно DefaultConfigKeys и DefaultConfigValues. Обычно это всё задаётся в одном месте, в данном случае application.conf файле. У вас получилось очень размазано.
Сам файл поддерживает возможность задавать дефолтные значения, хардкодить их где-то еще не надо. Также нужно оставить возможность переопределять их переменными среды (environment variables).
Более того, хост и порт из config объекта который сюда передаётся нигде не используются! У меня блин час ушёл пока настраивал докер и не мог понять почему адрес не переопределяется :)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DefaultConfugKeys используется, чтобы вычитать из application.conf значения. DefaultConfigValues используется, чтобы подставить дефолтное значение, если в application.conf что-то не задано. А так все вычитывается из application.conf, да

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Мой поинт в том, что выносить в переменные значения ключей из application.conf не стоит. Также не стоит хранить дефолтные значения в переменных, синтаксис application.conf позволяет прописать их прямо там. Ну и самое странное, что некоторые из настроек которые вы задали вообще не используются.


val mongoClient = MongoClient()
val mongoDB = mongoClient(config.dbName)

implicit val options: DBObject = BasicDBObjectBuilder.start().add("capped", true).add("size", 2000000000l).get()

def songsCollection =
if (mongoDB.collectionExists(config.songsCollection)) {
mongoDB(config.songsCollection)
} else {
mongoDB.createCollection(config.songsCollection, options)
mongoDB(config.songsCollection)
}

def wdCollection = {
if (mongoDB.collectionExists(config.wdCollection)) {
mongoDB(config.wdCollection)
} else {
mongoDB.createCollection(config.wdCollection, options)
mongoDB(config.wdCollection)
}
}

def drop = {
mongoDB.dropDatabase()
}



}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package scalalab3.lyricsengine.persistence.mongo.constants

object DefaultConfigKeys {
val mongo = "mongo"
val host = s"$mongo.host"
val port = s"$mongo.port"
val user = s"$mongo.user"
val dbname = s"$mongo.dbname"
val password = s"$mongo.password"
val songsCollection = s"$mongo.songscollection"
val wordsDefinitions = s"$mongo.wordsdefinitions"

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package scalalab3.lyricsengine.persistence.mongo.constants

object DefaultConfigValues {
val defaultHost = "localhost"
val defaultPort = 27017
val defaultUser = "admin"
val defaultPassword = ""
val defaultDbName = "lyrics"
val defaultSongsCollection = "songs"
val defaultWDCollection = "songs"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package scalalab3.lyricsengine.persistence

import java.util.UUID
import org.specs2.mutable.Specification
import org.specs2.specification._
import scala.util.Try
import scalalab3.lyricsengine.domain.{DataSet, Song}
import scalalab3.lyricsengine.persistence.mongo.{MongoConfig, MongoContext}

class StorageComponentImpl$Test extends Specification with BeforeAfterAll {
sequential

val tryMongoContext = Try(new MongoContext(MongoConfig.load()))

"MongoDB Test" >> {
if (tryMongoContext.isSuccess) {
implicit val s = tryMongoContext.get
val wd = Map(1 -> "i", 2 -> "the", 3 -> "you", 4 -> "to", 5 -> "and")
val firstSong = Song("TRZZZYV128F92E996D", "6849828", Map(1 -> 10, 2 -> 6, 3 -> 20, 5 -> 2, 7 -> 30))
val secondSong = Song("TRZZZYX128F92D32C6", "681124", Map(1 -> 4, 2 -> 18, 4 -> 3, 5 -> 6, 6 -> 9))
val seqSong = Seq(firstSong, secondSong)
val MdataSet=DataSet(wd, seqSong)
val mongoStorage = new StorageComponentImpl {
override val storage: Storage = new StorageImpl
}.storage

"add DataSet " in {
mongoStorage.addDataSet(MdataSet)
mongoStorage.getDataSet().songs.size must_== seqSong.size
mongoStorage.getDataSet().definition.size must_== wd.size
}

"get default LastVersion" in {
mongoStorage.getLastVersion must_== -1
}

"add DataSet with version " in {
mongoStorage.addDataSet(MdataSet, Some(1))
mongoStorage.getDataSet().songs.size must_== seqSong.size
mongoStorage.getDataSet().definition.size must_== wd.size
}

"get LastVersion" in {
mongoStorage.getLastVersion must_== 1
}

} else "Skipped Test" >> skipped("Mongo context is not available in ")
}

def uuid() = Some(UUID.randomUUID())

def drop() = for (m <- tryMongoContext) m.drop

override def beforeAll(): Unit = drop()
override def afterAll(): Unit = drop()
}