Scrape GitHub

app/db/impl/schema/ProjectSettingsTable.scala

@@ -14,9 +14,12 @@ class ProjectSettingsTable(tag: Tag) extends ModelTable[ProjectSettings](tag, "p
   def licenseName = column[String]("license_name")
   def licenseUrl  = column[String]("license_url")
   def forumSync   = column[Boolean]("forum_sync")
+  def githubSync  = column[Boolean]("github_sync")
 
   override def * =
-    mkProj((id.?, createdAt.?, projectId, homepage.?, issues.?, source.?, licenseName.?, licenseUrl.?, forumSync))(
+    mkProj(
+      (id.?, createdAt.?, projectId, homepage.?, issues.?, source.?, licenseName.?, licenseUrl.?, forumSync, githubSync)
+    )(
       mkTuple[ProjectSettings]()
     )
 }

app/form/OreForms.scala

@@ -102,7 +102,8 @@ class OreForms @Inject()(implicit config: OreConfig, factory: ProjectFactory, se
         "roleUps"      -> list(text),
         "update-icon"  -> boolean,
         "owner"        -> optional(longNumber).verifying(ownerIdInList(organisationUserCanUploadTo)),
-        "forum-sync"   -> boolean
+        "forum-sync"   -> boolean,
+        "github-sync"  -> boolean
       )(ProjectSettingsForm.apply)(ProjectSettingsForm.unapply)
     )
 

app/form/project/ProjectSettingsForm.scala

@@ -20,5 +20,6 @@ case class ProjectSettingsForm(
     roleUps: List[String],
     updateIcon: Boolean,
     ownerId: Option[DbRef[User]],
-    forumSync: Boolean
+    forumSync: Boolean,
+    githubSync: Boolean
 ) extends TProjectRoleSetBuilder

app/models/project/Project.scala

@@ -6,6 +6,7 @@ import java.time.Instant
 import play.api.i18n.Messages
 import play.api.libs.functional.syntax._
 import play.api.libs.json._
+import play.api.libs.ws.WSClient
 import play.twirl.api.Html
 
 import db.access.{ModelAccess, ModelAssociationAccess, ModelAssociationAccessImpl}
@@ -31,11 +32,12 @@ import ore.permission.scope.HasScope
 import ore.project.{Category, FlagReason, ProjectMember}
 import ore.user.MembershipDossier
 import ore.{Joinable, OreConfig, Visitable}
+import _root_.util.GitHubUtil
 import _root_.util.StringUtils
 import _root_.util.StringUtils._
 import _root_.util.syntax._
 
-import cats.data.OptionT
+import cats.data.{EitherT, OptionT}
 import cats.effect.{ContextShift, IO}
 import cats.syntax.all._
 import com.google.common.base.Preconditions._
@@ -331,7 +333,7 @@ case class Project(
 
   private def getOrInsert(name: String, parentId: Option[DbRef[Page]])(
       page: InsertFunc[Page]
-  )(implicit service: ModelService): IO[Page] = {
+  )(implicit service: ModelService): IO[(Page, Boolean)] = {
     def like =
       service.find[Page] { p =>
         p.projectId === this.id.value && p.name.toLowerCase === name.toLowerCase && parentId.fold(
@@ -340,21 +342,49 @@ case class Project(
       }
 
     like.value.flatMap {
-      case Some(u) => IO.pure(u)
-      case None    => service.insert(page)
+      case Some(u) => IO.pure((u, false))
+      case None    => service.insert(page).tupleRight(true)
     }
   }
 
+  def getGithubReadme(implicit service: ModelService, config: OreConfig, ws: WSClient): EitherT[IO, String, String] =
+    OptionT(this.settings.map(_.source))
+      .filter(GitHubUtil.isGitHubUrl)
+      .toRight("No github source")
+      .flatMap { githubSource =>
+        val urlParts  = githubSource.split("//github.com/", 2)(1).split("/")
+        val ghUser    = urlParts(0)
+        val ghProject = urlParts(1)
+        GitHubUtil.getReadme(ghUser, ghProject)
+      }
+
+  def syncHomepage(implicit service: ModelService, config: OreConfig, ws: WSClient): IO[Page] =
+    homePageOrCreate(scrapGithub = true).flatMap {
+      case (page, true) => IO.pure(page)
+      case (page, false) =>
+        getGithubReadme.semiflatMap(str => service.update(page.copy(contents = str))).getOrElse(page)
+    }
+
+  def homePageOrCreate(
+      scrapGithub: Boolean
+  )(implicit service: ModelService, config: OreConfig, ws: WSClient): IO[(Page, Boolean)] =
+    EitherT
+      .rightT[IO, String](scrapGithub)
+      .ensure("")(identity)
+      .flatMap(_ => getGithubReadme)
+      .getOrElse(Page.homeMessage)
+      .map { body =>
+        Page.partial(this.id.value, Page.homeName, Page.template(this.name, body), isDeletable = false, None)
+      }
+      .flatMap(page => getOrInsert(Page.homeName, None)(page))
+
   /**
     * Returns this Project's home page.
     *
     * @return Project home page
     */
-  def homePage(implicit service: ModelService, config: OreConfig): IO[Page] = {
-    val page =
-      Page.partial(this.id.value, Page.homeName, Page.template(this.name, Page.homeMessage), isDeletable = false, None)
-    getOrInsert(Page.homeName, None)(page)
-  }
+  def homePage(implicit service: ModelService, config: OreConfig, ws: WSClient): IO[Page] =
+    settings.flatMap(settings => homePageOrCreate(settings.githubSync)).map(_._1)
 
   /**
     * Returns true if a page with the specified name exists.
@@ -385,7 +415,7 @@ case class Project(
         text
     }
     val page = Page.partial(this.id.value, name, c, isDeletable = true, parentId)
-    getOrInsert(name, parentId)(page)
+    getOrInsert(name, parentId)(page).map(_._1)
   }
 
   /**

app/models/project/ProjectSettings.scala

@@ -14,6 +14,7 @@ import ore.project.factory.PendingProject
 import ore.project.io.ProjectFiles
 import ore.project.{Category, ProjectOwned}
 import ore.user.notification.NotificationType
+import util.GitHubUtil
 import util.StringUtils._
 
 import cats.data.NonEmptyList
@@ -42,7 +43,8 @@ case class ProjectSettings(
     source: Option[String],
     licenseName: Option[String],
     licenseUrl: Option[String],
-    forumSync: Boolean
+    forumSync: Boolean,
+    githubSync: Boolean
 ) extends Model {
 
   override type M = ProjectSettings
@@ -83,7 +85,8 @@ case class ProjectSettings(
         source = noneIfEmpty(formData.source),
         licenseUrl = noneIfEmpty(formData.licenseUrl),
         licenseName = if (formData.licenseUrl.nonEmpty) Some(formData.licenseName) else licenseName,
-        forumSync = formData.forumSync
+        forumSync = formData.forumSync,
+        githubSync = formData.githubSync
       )
     )
 
@@ -159,7 +162,8 @@ object ProjectSettings {
       source: Option[String] = None,
       licenseName: Option[String] = None,
       licenseUrl: Option[String] = None,
-      forumSync: Boolean = true
+      forumSync: Boolean = true,
+      githubSync: Boolean = true
   ) {
 
     /**
@@ -215,7 +219,19 @@ object ProjectSettings {
     }
 
     def asFunc(projectId: DbRef[Project]): InsertFunc[ProjectSettings] =
-      (id, time) => ProjectSettings(id, time, projectId, homepage, issues, source, licenseName, licenseUrl, forumSync)
+      (id, time) =>
+        ProjectSettings(
+          id,
+          time,
+          projectId,
+          homepage,
+          issues,
+          source,
+          licenseName,
+          licenseUrl,
+          forumSync,
+          githubSync && source.exists(GitHubUtil.isGitHubUrl)
+      )
   }
 
   implicit val query: ModelQuery[ProjectSettings] =

app/ore/project/ProjectTask.scala

@@ -7,8 +7,11 @@ import javax.inject.{Inject, Singleton}
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 
+import play.api.libs.ws.WSClient
+
 import db.ModelFilter._
 import db.impl.OrePostgresDriver.api._
+import db.impl.schema.{ProjectSettingsTable, ProjectTableMain}
 import db.{ModelFilter, ModelService}
 import models.project.{Project, Visibility}
 import ore.OreConfig
@@ -21,7 +24,7 @@ import com.typesafe.scalalogging
   * Task that is responsible for publishing New projects
   */
 @Singleton
-class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig)(
+class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig, ws: WSClient)(
     implicit ec: ExecutionContext,
     service: ModelService
 ) extends Runnable {
@@ -38,6 +41,12 @@ class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig)(
   private def createdAtFilter = ModelFilter[Project](_.createdAt < dayAgo)
   private def newProjects     = service.filter[Project](newFilter && createdAtFilter)
 
+  private val githubSyncProjects = for {
+    project  <- TableQuery[ProjectTableMain]
+    settings <- TableQuery[ProjectSettingsTable] if settings.id === project.id
+    if settings.githubSync
+  } yield project
+
   /**
     * Starts the task.
     */
@@ -49,10 +58,19 @@ class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig)(
   /**
     * Task runner
     */
-  def run(): Unit = newProjects.unsafeToFuture().foreach { projects =>
-    projects.foreach { project =>
-      Logger.debug(s"Changed ${project.ownerName}/${project.slug} from New to Public")
-      project.setVisibility(Visibility.Public, "Changed by task", project.ownerId).unsafeRunAsyncAndForget()
+  def run(): Unit = {
+    newProjects.unsafeToFuture().foreach { projects =>
+      projects.foreach { project =>
+        Logger.debug(s"Changed ${project.ownerName}/${project.slug} from New to Public")
+        project.setVisibility(Visibility.Public, "Changed by task", project.ownerId).unsafeRunAsyncAndForget()
+      }
+    }
+
+    service.runDBIO(githubSyncProjects.result).unsafeToFuture().foreach { projects =>
+      projects.foreach { project =>
+        Logger.debug(s"Syncing README for ${project.ownerName}/${project.slug} from Github")
+        project.syncHomepage(service, config, ws).unsafeRunAsyncAndForget()
+      }
     }
   }
 }

app/util/GitHubUtil.scala

@@ -0,0 +1,32 @@
+package util
+
+import java.util.Base64
+
+import play.api.libs.ws.WSClient
+
+import cats.data.EitherT
+import cats.effect.IO
+import cats.syntax.all._
+
+object GitHubUtil {
+
+  private val identifier       = "A-Za-z0-9-_"
+  private val gitHubUrlPattern = s"""http(s)?://github.com/[$identifier]+/[$identifier]+(/)?""".r.pattern
+  private val readmeApi        = "https://api.github.com/repos/%s/%s/readme"
+
+  def isGitHubUrl(url: String): Boolean = gitHubUrlPattern.matcher(url).matches()
+
+  def getReadme(user: String, project: String)(implicit ws: WSClient): EitherT[IO, String, String] =
+    EitherT(
+      IO.fromFuture(IO(ws.url(readmeApi.format(user, project)).get())).map { res =>
+        if (res.status == 200) {
+          (res.json \ "content")
+            .validate[String]
+            .asEither
+            .leftMap(
+              _.map(t => s"Failed to decode ${t._1.path} because ${t._2.map(_.message).mkString("\n")}").mkString("\n")
+            )
+        } else Left(res.body)
+      }
+    ).map(content => new String(Base64.getDecoder.decode(content.replace("\\n", "")), "UTF-8"))
+}

app/views/projects/helper/inputSettings.scala.html

@@ -6,7 +6,8 @@
   licenseName: Option[String] = None,
   licenseUrl: Option[String] = None,
   selected: Option[Category] = None,
-  forumSync: Boolean = true)(implicit messages: Messages)
+  forumSync: Boolean = true,
+  githubSync: Boolean = true)(implicit messages: Messages)
 
 <script type="text/javascript" src="@routes.Assets.versioned("javascripts/projectManage.js")"></script>
 
@@ -96,3 +97,17 @@ <h4>@messages("project.settings.forumSync")</h4>
     </div>
     <div class="clearfix"></div>
 </div>
+
+<div class="setting">
+    <div class="setting-description">
+        <h4>@messages("project.settings.githubSync")</h4>
+        <p>@messages("project.settings.githubSync.info")</p>
+    </div>
+    <div class="setting-content">
+        <label>
+            <input @if(githubSync) { checked } value="true" form="@form" type="checkbox" id="github-sync" name="github-sync">
+            Sync github README
+        </label>
+    </div>
+    <div class="clearfix"></div>
+</div>

app/views/projects/settings.scala.html

@@ -65,7 +65,8 @@ <h4 class="modal-title" style="color:black;">Comment</h4>
                             licenseName = p.settings.licenseName,
                             licenseUrl = p.settings.licenseUrl,
                             selected = Some(p.project.category),
-                            forumSync = p.settings.forumSync
+                            forumSync = p.settings.forumSync,
+                            githubSync = p.settings.githubSync
                         )
 
                         <!-- Description -->

conf/evolutions/default/112.sql

@@ -0,0 +1,8 @@
+# --- !Ups
+
+ALTER TABLE project_settings ADD COLUMN github_sync BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE project_settings ALTER COLUMN github_sync DROP DEFAULT;
+
+# --- !Downs
+
+ALTER TABLE project_settings DROP COLUMN github_sync;

conf/messages

@@ -165,6 +165,8 @@ project.settings.genKey         =   Generate key
 project.settings.revokeKey      =   Revoke key
 project.settings.forumSync      =   Create posts on the forums
 project.settings.forumSync.info =   Sets if events like a new release should automatically create a post on the forums
+project.settings.githubSync     =   Sync Github README
+project.settings.githubSync.info=   Regularly syncs the main project page with the Github README. Requires that you supply a Github source.
 project.versions                =   Versions
 project.downloads               =   Downloads
 project.starred                 =   Stars