diff --git a/app/db/impl/schema/ProjectSettingsTable.scala b/app/db/impl/schema/ProjectSettingsTable.scala index ed11607c9..e5dd28993 100644 --- a/app/db/impl/schema/ProjectSettingsTable.scala +++ b/app/db/impl/schema/ProjectSettingsTable.scala @@ -14,9 +14,12 @@ class ProjectSettingsTable(tag: Tag) extends ModelTable[ProjectSettings](tag, "p def licenseName = column[String]("license_name") def licenseUrl = column[String]("license_url") def forumSync = column[Boolean]("forum_sync") + def githubSync = column[Boolean]("github_sync") override def * = - mkProj((id.?, createdAt.?, projectId, homepage.?, issues.?, source.?, licenseName.?, licenseUrl.?, forumSync))( + mkProj( + (id.?, createdAt.?, projectId, homepage.?, issues.?, source.?, licenseName.?, licenseUrl.?, forumSync, githubSync) + )( mkTuple[ProjectSettings]() ) } diff --git a/app/form/OreForms.scala b/app/form/OreForms.scala index 2d8b9c61b..9eb183406 100755 --- a/app/form/OreForms.scala +++ b/app/form/OreForms.scala @@ -102,7 +102,8 @@ class OreForms @Inject()(implicit config: OreConfig, factory: ProjectFactory, se "roleUps" -> list(text), "update-icon" -> boolean, "owner" -> optional(longNumber).verifying(ownerIdInList(organisationUserCanUploadTo)), - "forum-sync" -> boolean + "forum-sync" -> boolean, + "github-sync" -> boolean )(ProjectSettingsForm.apply)(ProjectSettingsForm.unapply) ) diff --git a/app/form/project/ProjectSettingsForm.scala b/app/form/project/ProjectSettingsForm.scala index fd2b7b665..e6d465823 100644 --- a/app/form/project/ProjectSettingsForm.scala +++ b/app/form/project/ProjectSettingsForm.scala @@ -20,5 +20,6 @@ case class ProjectSettingsForm( roleUps: List[String], updateIcon: Boolean, ownerId: Option[DbRef[User]], - forumSync: Boolean + forumSync: Boolean, + githubSync: Boolean ) extends TProjectRoleSetBuilder diff --git a/app/models/project/Project.scala b/app/models/project/Project.scala index 4fccaa2a4..b42235948 100644 --- a/app/models/project/Project.scala +++ b/app/models/project/Project.scala @@ -6,6 +6,7 @@ import java.time.Instant import play.api.i18n.Messages import play.api.libs.functional.syntax._ import play.api.libs.json._ +import play.api.libs.ws.WSClient import play.twirl.api.Html import db.access.{ModelAccess, ModelAssociationAccess, ModelAssociationAccessImpl} @@ -31,11 +32,12 @@ import ore.permission.scope.HasScope import ore.project.{Category, FlagReason, ProjectMember} import ore.user.MembershipDossier import ore.{Joinable, OreConfig, Visitable} +import _root_.util.GitHubUtil import _root_.util.StringUtils import _root_.util.StringUtils._ import _root_.util.syntax._ -import cats.data.OptionT +import cats.data.{EitherT, OptionT} import cats.effect.{ContextShift, IO} import cats.syntax.all._ import com.google.common.base.Preconditions._ @@ -331,7 +333,7 @@ case class Project( private def getOrInsert(name: String, parentId: Option[DbRef[Page]])( page: InsertFunc[Page] - )(implicit service: ModelService): IO[Page] = { + )(implicit service: ModelService): IO[(Page, Boolean)] = { def like = service.find[Page] { p => p.projectId === this.id.value && p.name.toLowerCase === name.toLowerCase && parentId.fold( @@ -340,21 +342,49 @@ case class Project( } like.value.flatMap { - case Some(u) => IO.pure(u) - case None => service.insert(page) + case Some(u) => IO.pure((u, false)) + case None => service.insert(page).tupleRight(true) } } + def getGithubReadme(implicit service: ModelService, config: OreConfig, ws: WSClient): EitherT[IO, String, String] = + OptionT(this.settings.map(_.source)) + .filter(GitHubUtil.isGitHubUrl) + .toRight("No github source") + .flatMap { githubSource => + val urlParts = githubSource.split("//github.com/", 2)(1).split("/") + val ghUser = urlParts(0) + val ghProject = urlParts(1) + GitHubUtil.getReadme(ghUser, ghProject) + } + + def syncHomepage(implicit service: ModelService, config: OreConfig, ws: WSClient): IO[Page] = + homePageOrCreate(scrapGithub = true).flatMap { + case (page, true) => IO.pure(page) + case (page, false) => + getGithubReadme.semiflatMap(str => service.update(page.copy(contents = str))).getOrElse(page) + } + + def homePageOrCreate( + scrapGithub: Boolean + )(implicit service: ModelService, config: OreConfig, ws: WSClient): IO[(Page, Boolean)] = + EitherT + .rightT[IO, String](scrapGithub) + .ensure("")(identity) + .flatMap(_ => getGithubReadme) + .getOrElse(Page.homeMessage) + .map { body => + Page.partial(this.id.value, Page.homeName, Page.template(this.name, body), isDeletable = false, None) + } + .flatMap(page => getOrInsert(Page.homeName, None)(page)) + /** * Returns this Project's home page. * * @return Project home page */ - def homePage(implicit service: ModelService, config: OreConfig): IO[Page] = { - val page = - Page.partial(this.id.value, Page.homeName, Page.template(this.name, Page.homeMessage), isDeletable = false, None) - getOrInsert(Page.homeName, None)(page) - } + def homePage(implicit service: ModelService, config: OreConfig, ws: WSClient): IO[Page] = + settings.flatMap(settings => homePageOrCreate(settings.githubSync)).map(_._1) /** * Returns true if a page with the specified name exists. @@ -385,7 +415,7 @@ case class Project( text } val page = Page.partial(this.id.value, name, c, isDeletable = true, parentId) - getOrInsert(name, parentId)(page) + getOrInsert(name, parentId)(page).map(_._1) } /** diff --git a/app/models/project/ProjectSettings.scala b/app/models/project/ProjectSettings.scala index 9ced3cd07..23b2b436c 100644 --- a/app/models/project/ProjectSettings.scala +++ b/app/models/project/ProjectSettings.scala @@ -14,6 +14,7 @@ import ore.project.factory.PendingProject import ore.project.io.ProjectFiles import ore.project.{Category, ProjectOwned} import ore.user.notification.NotificationType +import util.GitHubUtil import util.StringUtils._ import cats.data.NonEmptyList @@ -42,7 +43,8 @@ case class ProjectSettings( source: Option[String], licenseName: Option[String], licenseUrl: Option[String], - forumSync: Boolean + forumSync: Boolean, + githubSync: Boolean ) extends Model { override type M = ProjectSettings @@ -83,7 +85,8 @@ case class ProjectSettings( source = noneIfEmpty(formData.source), licenseUrl = noneIfEmpty(formData.licenseUrl), licenseName = if (formData.licenseUrl.nonEmpty) Some(formData.licenseName) else licenseName, - forumSync = formData.forumSync + forumSync = formData.forumSync, + githubSync = formData.githubSync ) ) @@ -159,7 +162,8 @@ object ProjectSettings { source: Option[String] = None, licenseName: Option[String] = None, licenseUrl: Option[String] = None, - forumSync: Boolean = true + forumSync: Boolean = true, + githubSync: Boolean = true ) { /** @@ -215,7 +219,19 @@ object ProjectSettings { } def asFunc(projectId: DbRef[Project]): InsertFunc[ProjectSettings] = - (id, time) => ProjectSettings(id, time, projectId, homepage, issues, source, licenseName, licenseUrl, forumSync) + (id, time) => + ProjectSettings( + id, + time, + projectId, + homepage, + issues, + source, + licenseName, + licenseUrl, + forumSync, + githubSync && source.exists(GitHubUtil.isGitHubUrl) + ) } implicit val query: ModelQuery[ProjectSettings] = diff --git a/app/ore/project/ProjectTask.scala b/app/ore/project/ProjectTask.scala index c08fc12a5..9708fcf6d 100644 --- a/app/ore/project/ProjectTask.scala +++ b/app/ore/project/ProjectTask.scala @@ -7,8 +7,11 @@ import javax.inject.{Inject, Singleton} import scala.concurrent.ExecutionContext import scala.concurrent.duration._ +import play.api.libs.ws.WSClient + import db.ModelFilter._ import db.impl.OrePostgresDriver.api._ +import db.impl.schema.{ProjectSettingsTable, ProjectTableMain} import db.{ModelFilter, ModelService} import models.project.{Project, Visibility} import ore.OreConfig @@ -21,7 +24,7 @@ import com.typesafe.scalalogging * Task that is responsible for publishing New projects */ @Singleton -class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig)( +class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig, ws: WSClient)( implicit ec: ExecutionContext, service: ModelService ) extends Runnable { @@ -38,6 +41,12 @@ class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig)( private def createdAtFilter = ModelFilter[Project](_.createdAt < dayAgo) private def newProjects = service.filter[Project](newFilter && createdAtFilter) + private val githubSyncProjects = for { + project <- TableQuery[ProjectTableMain] + settings <- TableQuery[ProjectSettingsTable] if settings.id === project.id + if settings.githubSync + } yield project + /** * Starts the task. */ @@ -49,10 +58,19 @@ class ProjectTask @Inject()(actorSystem: ActorSystem, config: OreConfig)( /** * Task runner */ - def run(): Unit = newProjects.unsafeToFuture().foreach { projects => - projects.foreach { project => - Logger.debug(s"Changed ${project.ownerName}/${project.slug} from New to Public") - project.setVisibility(Visibility.Public, "Changed by task", project.ownerId).unsafeRunAsyncAndForget() + def run(): Unit = { + newProjects.unsafeToFuture().foreach { projects => + projects.foreach { project => + Logger.debug(s"Changed ${project.ownerName}/${project.slug} from New to Public") + project.setVisibility(Visibility.Public, "Changed by task", project.ownerId).unsafeRunAsyncAndForget() + } + } + + service.runDBIO(githubSyncProjects.result).unsafeToFuture().foreach { projects => + projects.foreach { project => + Logger.debug(s"Syncing README for ${project.ownerName}/${project.slug} from Github") + project.syncHomepage(service, config, ws).unsafeRunAsyncAndForget() + } } } } diff --git a/app/util/GitHubUtil.scala b/app/util/GitHubUtil.scala new file mode 100644 index 000000000..a21e5e828 --- /dev/null +++ b/app/util/GitHubUtil.scala @@ -0,0 +1,32 @@ +package util + +import java.util.Base64 + +import play.api.libs.ws.WSClient + +import cats.data.EitherT +import cats.effect.IO +import cats.syntax.all._ + +object GitHubUtil { + + private val identifier = "A-Za-z0-9-_" + private val gitHubUrlPattern = s"""http(s)?://github.com/[$identifier]+/[$identifier]+(/)?""".r.pattern + private val readmeApi = "https://api.github.com/repos/%s/%s/readme" + + def isGitHubUrl(url: String): Boolean = gitHubUrlPattern.matcher(url).matches() + + def getReadme(user: String, project: String)(implicit ws: WSClient): EitherT[IO, String, String] = + EitherT( + IO.fromFuture(IO(ws.url(readmeApi.format(user, project)).get())).map { res => + if (res.status == 200) { + (res.json \ "content") + .validate[String] + .asEither + .leftMap( + _.map(t => s"Failed to decode ${t._1.path} because ${t._2.map(_.message).mkString("\n")}").mkString("\n") + ) + } else Left(res.body) + } + ).map(content => new String(Base64.getDecoder.decode(content.replace("\\n", "")), "UTF-8")) +} diff --git a/app/views/projects/helper/inputSettings.scala.html b/app/views/projects/helper/inputSettings.scala.html index 3908fe2e7..86b68b59d 100644 --- a/app/views/projects/helper/inputSettings.scala.html +++ b/app/views/projects/helper/inputSettings.scala.html @@ -6,7 +6,8 @@ licenseName: Option[String] = None, licenseUrl: Option[String] = None, selected: Option[Category] = None, - forumSync: Boolean = true)(implicit messages: Messages) + forumSync: Boolean = true, + githubSync: Boolean = true)(implicit messages: Messages) @@ -96,3 +97,17 @@
@messages("project.settings.githubSync.info")
+