Merge pull request #245 from lichess-org/index-full-forum-posts

Index full forum posts
lichess-org · Jun 22, 2024 · 2eaa27f · 2eaa27f
2 parents 2f8f242 + f5ef97b
commit 2eaa27f
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 5 deletions.
diff --git a/modules/ingestor/src/main/scala/app.config.scala b/modules/ingestor/src/main/scala/app.config.scala
@@ -38,22 +38,20 @@ object ElasticConfig:
 case class IngestorConfig(forum: IngestorConfig.Forum, team: IngestorConfig.Team)
 
 object IngestorConfig:
-  case class Forum(batchSize: Int, maxBodyLength: Int, timeWindows: Int, startAt: Option[Long])
+  case class Forum(batchSize: Int, timeWindows: Int, startAt: Option[Long])
   case class Team(batchSize: Int, timeWindows: Int, startAt: Option[Long])
 
   private object Forum:
     private def batchSize =
       env("INGESTOR_FORUM_BATCH_SIZE").or(prop("ingestor.forum.batch.size")).as[Int].default(100)
-    private def maxBodyLength =
-      env("INGESTOR_FORUM_MAX_BODY_LENGTH").or(prop("ingestor.forum.max.body.length")).as[Int].default(10000)
     private def timeWindows =
       env("INGESTOR_FORUM_TIME_WINDOWS").or(prop("ingestor.forum.time.windows")).as[Int].default(10)
     private def startAt =
       env("INGESTOR_FORUM_START_AT")
         .or(prop("ingestor.forum.start.at"))
         .as[Long]
         .option
-    def config = (batchSize, maxBodyLength, timeWindows, startAt).parMapN(Forum.apply)
+    def config = (batchSize, timeWindows, startAt).parMapN(Forum.apply)
 
   private object Team:
     private def batchSize =

diff --git a/modules/ingestor/src/main/scala/ingestor.forum.scala b/modules/ingestor/src/main/scala/ingestor.forum.scala
@@ -170,7 +170,7 @@ object ForumIngestor:
 
       private def toSource(topicName: Option[String], topicId: String): Option[ForumSource] =
         (
-          doc.getString("text").map(_.take(config.maxBodyLength)),
+          doc.getString("text"),
           topicName,
           doc.getBoolean("troll"),
           doc.getNested("createdAt").flatMap(_.asInstant).map(_.toEpochMilli()),