diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index c405c71fe34..0b6227b564c 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -13,6 +13,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Added - WEBKNOSSOS now automatically searches in subfolder / sub-collection identifiers for valid datasets in case a provided link to a remote dataset does not directly point to a dataset. [#7912](https://github.com/scalableminds/webknossos/pull/7912) - Added the option to move a bounding box via dragging while pressing ctrl / meta. [#7892](https://github.com/scalableminds/webknossos/pull/7892) +- Added the option to add metadata entries to datasets and folders. The metadata can be viewed and edited in the dashboard in the right details tab.[#7886](https://github.com/scalableminds/webknossos/pull/7886) - Added route `/import?url=` to automatically import and view remote datasets. [#7844](https://github.com/scalableminds/webknossos/pull/7844) - Added that newly created, modified and clicked on bounding boxes are now highlighted and scrolled into view, while the bounding box tool is active. [#7935](https://github.com/scalableminds/webknossos/pull/7935) - The configured unit in the dataset upload view is now passed to the convert_to_wkw worker job. [#7970](https://github.com/scalableminds/webknossos/pull/7970) diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md index 38c39ea21da..94cc682c67e 100644 --- a/MIGRATIONS.unreleased.md +++ b/MIGRATIONS.unreleased.md @@ -11,3 +11,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md). - If Segment Anything was already configured, it needs to be pointed to an endpoint that works with SAM 2. [#7965](https://github.com/scalableminds/webknossos/pull/7965) ### Postgres Evolutions: +- [119-add-metadata-to-folders-and-datasets.sql](conf/evolutions/119-add-metadata-to-folders-and-datasets.sql) diff --git a/app/controllers/DatasetController.scala b/app/controllers/DatasetController.scala index a92b0e4e7de..e238b919c30 100755 --- a/app/controllers/DatasetController.scala +++ b/app/controllers/DatasetController.scala @@ -36,6 +36,7 @@ case class DatasetUpdateParameters( sortingKey: Option[Instant], isPublic: Option[Boolean], tags: Option[List[String]], + metadata: Option[JsArray], folderId: Option[ObjectId] ) @@ -94,6 +95,7 @@ class DatasetController @Inject()(userService: UserService, (__ \ "sortingKey").readNullable[Instant] and (__ \ "isPublic").read[Boolean] and (__ \ "tags").read[List[String]] and + (__ \ "metadata").readNullable[JsArray] and (__ \ "folderId").readNullable[ObjectId]).tupled def removeFromThumbnailCache(organizationName: String, datasetName: String): Action[AnyContent] = @@ -313,18 +315,22 @@ class DatasetController @Inject()(userService: UserService, def update(organizationName: String, datasetName: String): Action[JsValue] = sil.SecuredAction.async(parse.json) { implicit request => withJsonBodyUsing(datasetPublicReads) { - case (description, displayName, sortingKey, isPublic, tags, folderId) => + case (description, displayName, sortingKey, isPublic, tags, metadata, folderId) => for { dataset <- datasetDAO.findOneByNameAndOrganization(datasetName, request.identity._organization) ?~> notFoundMessage( datasetName) ~> NOT_FOUND + maybeUpdatedMetadata = metadata.getOrElse(dataset.metadata) _ <- Fox.assertTrue(datasetService.isEditableBy(dataset, Some(request.identity))) ?~> "notAllowed" ~> FORBIDDEN - _ <- datasetDAO.updateFields(dataset._id, - description, - displayName, - sortingKey.getOrElse(dataset.created), - isPublic, - folderId.getOrElse(dataset._folder)) - _ <- datasetDAO.updateTags(dataset._id, tags) + _ <- datasetDAO.updateFields( + dataset._id, + description, + displayName, + sortingKey.getOrElse(dataset.created), + isPublic, + tags, + maybeUpdatedMetadata, + folderId.getOrElse(dataset._folder) + ) updated <- datasetDAO.findOneByNameAndOrganization(datasetName, request.identity._organization) _ = analyticsService.track(ChangeDatasetSettingsEvent(request.identity, updated)) js <- datasetService.publicWrites(updated, Some(request.identity)) diff --git a/app/controllers/FolderController.scala b/app/controllers/FolderController.scala index fd28298d61f..e4f189b264f 100644 --- a/app/controllers/FolderController.scala +++ b/app/controllers/FolderController.scala @@ -7,7 +7,7 @@ import models.folder.{Folder, FolderDAO, FolderParameters, FolderService} import models.organization.OrganizationDAO import models.team.{TeamDAO, TeamService} import models.user.UserService -import play.api.libs.json.Json +import play.api.libs.json.{JsArray, Json} import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import security.WkEnv import utils.ObjectId @@ -53,6 +53,7 @@ class FolderController @Inject()( _ <- folderDAO.findOne(idValidated) ?~> "folder.notFound" - <- Fox.assertTrue(folderDAO.isEditable(idValidated)) ?~> "folder.update.notAllowed" ~> FORBIDDEN _ <- folderService.assertValidFolderName(params.name) + _ <- folderDAO.updateMetadata(idValidated, params.metadata) _ <- folderDAO.updateName(idValidated, params.name) ?~> "folder.update.name.failed" _ <- folderService .updateAllowedTeams(idValidated, params.allowedTeams, request.identity) ?~> "folder.update.teams.failed" @@ -103,7 +104,7 @@ class FolderController @Inject()( for { parentIdValidated <- ObjectId.fromString(parentId) _ <- folderService.assertValidFolderName(name) - newFolder = Folder(ObjectId.generate, name) + newFolder = Folder(ObjectId.generate, name, JsArray.empty) _ <- folderDAO.findOne(parentIdValidated) ?~> "folder.notFound" _ <- folderDAO.insertAsChild(parentIdValidated, newFolder) ?~> "folder.create.failed" organization <- organizationDAO.findOne(request.identity._organization) ?~> "folder.notFound" diff --git a/app/controllers/InitialDataController.scala b/app/controllers/InitialDataController.scala index 4dce4f72e03..01b230337e7 100644 --- a/app/controllers/InitialDataController.scala +++ b/app/controllers/InitialDataController.scala @@ -13,7 +13,7 @@ import models.task.{TaskType, TaskTypeDAO} import models.team._ import models.user._ import net.liftweb.common.{Box, Full} -import play.api.libs.json.Json +import play.api.libs.json.{JsArray, Json} import utils.{ObjectId, StoreModules, WkConf} import javax.inject.Inject @@ -169,7 +169,8 @@ Samplecountry private def insertRootFolder(): Fox[Unit] = folderDAO.findOne(defaultOrganization._rootFolder).futureBox.flatMap { case Full(_) => Fox.successful(()) - case _ => folderDAO.insertAsRoot(Folder(defaultOrganization._rootFolder, folderService.defaultRootName)) + case _ => + folderDAO.insertAsRoot(Folder(defaultOrganization._rootFolder, folderService.defaultRootName, JsArray.empty)) } private def insertDefaultUser(userEmail: String, diff --git a/app/models/dataset/Dataset.scala b/app/models/dataset/Dataset.scala index 444f1eaafde..9a012287fdd 100755 --- a/app/models/dataset/Dataset.scala +++ b/app/models/dataset/Dataset.scala @@ -55,7 +55,7 @@ case class Dataset(_id: ObjectId, status: String, logoUrl: Option[String], sortingKey: Instant = Instant.now, - details: Option[JsObject] = None, + metadata: JsArray = JsArray.empty, tags: List[String] = List.empty, created: Instant = Instant.now, isDeleted: Boolean = false) @@ -119,7 +119,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA JsonHelper.parseAndValidateJson[DatasetViewConfiguration](_)) adminViewConfigurationOpt <- Fox.runOptional(r.adminviewconfiguration)( JsonHelper.parseAndValidateJson[DatasetViewConfiguration](_)) - details <- Fox.runOptional(r.details)(JsonHelper.parseAndValidateJson[JsObject](_)) + metadata <- JsonHelper.parseAndValidateJson[JsArray](r.metadata) } yield { Dataset( ObjectId(r._Id), @@ -141,7 +141,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA r.status, r.logourl, Instant.fromSql(r.sortingkey), - details, + metadata, parseArrayLiteral(r.tags).sorted, Instant.fromSql(r.created), r.isdeleted @@ -493,6 +493,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA params.isPublic.map(v => q"isPublic = $v"), params.tags.map(v => q"tags = $v"), params.folderId.map(v => q"_folder = $v"), + params.metadata.map(v => q"metadata = $v"), ).flatten if (setQueries.isEmpty) { Fox.successful(()) @@ -509,18 +510,24 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA } } - def updateFields(_id: ObjectId, + def updateFields(datasetId: ObjectId, description: Option[String], displayName: Option[String], sortingKey: Instant, isPublic: Boolean, + tags: List[String], + metadata: JsArray, folderId: ObjectId)(implicit ctx: DBAccessContext): Fox[Unit] = { - val query = for { row <- Datasets if notdel(row) && row._Id === _id.id } yield - (row.description, row.displayname, row.sortingkey, row.ispublic, row._Folder) - for { - _ <- assertUpdateAccess(_id) - _ <- run(query.update(description, displayName, sortingKey.toSql, isPublic, folderId.toString)) - } yield () + val updateParameters = new DatasetUpdateParameters( + description = Some(description), + displayName = Some(displayName), + sortingKey = Some(sortingKey), + isPublic = Some(isPublic), + tags = Some(tags), + metadata = Some(metadata), + folderId = Some(folderId) + ) + updatePartial(datasetId, updateParameters) } def updateTags(id: ObjectId, tags: List[String])(implicit ctx: DBAccessContext): Fox[Unit] = @@ -564,7 +571,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA inboxSourceHash, defaultViewConfiguration, adminViewConfiguration, description, displayName, isPublic, isUsable, name, voxelSizeFactor, voxelSizeUnit, status, - sharingToken, sortingKey, details, tags, + sharingToken, sortingKey, metadata, tags, created, isDeleted ) VALUES( @@ -573,7 +580,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA ${d.inboxSourceHash}, $defaultViewConfiguration, $adminViewConfiguration, ${d.description}, ${d.displayName}, ${d.isPublic}, ${d.isUsable}, ${d.name}, ${d.voxelSize.map(_.factor)}, ${d.voxelSize.map(_.unit)}, ${d.status.take(1024)}, - ${d.sharingToken}, ${d.sortingKey}, ${d.details}, ${d.tags}, + ${d.sharingToken}, ${d.sortingKey}, ${d.metadata}, ${d.tags}, ${d.created}, ${d.isDeleted} )""".asUpdate) } yield () diff --git a/app/models/dataset/DatasetService.scala b/app/models/dataset/DatasetService.scala index e2bbc25426c..c8fe60fe126 100644 --- a/app/models/dataset/DatasetService.scala +++ b/app/models/dataset/DatasetService.scala @@ -90,8 +90,15 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO, ): Fox[Dataset] = { implicit val ctx: DBAccessContext = GlobalAccessContext val newId = ObjectId.generate - val details = - Json.obj("species" -> "species name", "brainRegion" -> "brain region", "acquisition" -> "acquisition method") + val metadata = + if (publication.isDefined) + Json.arr( + Json.obj("type" -> "string", "key" -> "species", "value" -> "species name"), + Json.obj("type" -> "string", "key" -> "brainRegion", "value" -> "brain region"), + Json.obj("type" -> "string", "key" -> "acquisition", "value" -> "acquisition method") + ) + else Json.arr() + val dataSourceHash = if (dataSource.isUsable) Some(dataSource.hashCode()) else None for { organization <- organizationDAO.findOneByName(owningOrganization) @@ -115,7 +122,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO, sharingToken = None, status = dataSource.statusOpt.getOrElse(""), logoUrl = None, - details = publication.map(_ => details) + metadata = metadata ) _ <- datasetDAO.insertOne(dataset) _ <- datasetDataLayerDAO.updateLayers(newId, dataSource) @@ -366,7 +373,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO, "lastUsedByUser" -> lastUsedByUser, "logoUrl" -> logoUrl, "sortingKey" -> dataset.sortingKey, - "details" -> dataset.details, + "metadata" -> dataset.metadata, "isUnreported" -> Json.toJson(isUnreported(dataset)), "tags" -> dataset.tags, "folderId" -> dataset._folder, diff --git a/app/models/folder/Folder.scala b/app/models/folder/Folder.scala index f60c5825e5b..730d8032325 100644 --- a/app/models/folder/Folder.scala +++ b/app/models/folder/Folder.scala @@ -1,14 +1,14 @@ package models.folder import com.scalableminds.util.accesscontext.DBAccessContext -import com.scalableminds.util.tools.Fox +import com.scalableminds.util.tools.{Fox, JsonHelper} import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox} import com.scalableminds.webknossos.schema.Tables._ import com.typesafe.scalalogging.LazyLogging import models.organization.{Organization, OrganizationDAO} import models.team.{TeamDAO, TeamService} import models.user.User -import play.api.libs.json.{JsObject, Json, OFormat} +import play.api.libs.json.{JsArray, JsObject, Json, OFormat} import slick.jdbc.PostgresProfile.api._ import slick.lifted.Rep import slick.sql.SqlAction @@ -19,11 +19,11 @@ import javax.inject.Inject import scala.annotation.tailrec import scala.concurrent.ExecutionContext -case class Folder(_id: ObjectId, name: String) +case class Folder(_id: ObjectId, name: String, metadata: JsArray) -case class FolderWithParent(_id: ObjectId, name: String, _parent: Option[ObjectId]) +case class FolderWithParent(_id: ObjectId, name: String, metadata: JsArray, _parent: Option[ObjectId]) -case class FolderParameters(name: String, allowedTeams: List[ObjectId]) +case class FolderParameters(name: String, allowedTeams: List[ObjectId], metadata: JsArray) object FolderParameters { implicit val jsonFormat: OFormat[FolderParameters] = Json.format[FolderParameters] } @@ -48,17 +48,21 @@ class FolderService @Inject()(teamDAO: TeamDAO, teamService.publicWrites(t, requestingUserOrganization)) ?~> "dataset.list.teamWritesFailed" isEditable <- folderDAO.isEditable(folder._id) } yield - Json.obj("id" -> folder._id, - "name" -> folder.name, - "allowedTeams" -> teamsJs, - "allowedTeamsCumulative" -> teamsCumulativeJs, - "isEditable" -> isEditable) + Json.obj( + "id" -> folder._id, + "name" -> folder.name, + "metadata" -> folder.metadata, + "allowedTeams" -> teamsJs, + "allowedTeamsCumulative" -> teamsCumulativeJs, + "isEditable" -> isEditable + ) def publicWritesWithParent(folderWithParent: FolderWithParent, allEditableIds: Set[ObjectId]): JsObject = Json.obj( "id" -> folderWithParent._id, "name" -> folderWithParent.name, "parent" -> folderWithParent._parent, + "metadata" -> folderWithParent.metadata, "isEditable" -> allEditableIds.contains(folderWithParent._id) ) @@ -117,7 +121,7 @@ class FolderService @Inject()(teamDAO: TeamDAO, remainingPathNames match { case pathNamesHead :: pathNamesTail => for { - newFolder <- Fox.successful(Folder(ObjectId.generate, pathNamesHead)) + newFolder <- Fox.successful(Folder(ObjectId.generate, pathNamesHead, JsArray.empty)) _ <- folderDAO.insertAsChild(parentFolderId, newFolder) folderId <- createMissingFoldersForPathNames(newFolder._id, pathNamesTail) } yield folderId @@ -133,10 +137,18 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) protected def isDeletedColumn(x: Folders): Rep[Boolean] = x.isdeleted protected def parse(r: FoldersRow): Fox[Folder] = - Fox.successful(Folder(ObjectId(r._Id), r.name)) + for { + metadata <- parseMetadata(r.metadata) + } yield Folder(ObjectId(r._Id), r.name, metadata) + + private def parseWithParent(t: (String, String, String, Option[String])): Fox[FolderWithParent] = + for { + metadata <- parseMetadata(t._3) + folderWithParent = FolderWithParent(ObjectId(t._1), t._2, metadata, t._4.map(ObjectId(_))) + } yield folderWithParent - private def parseWithParent(t: (String, String, Option[String])): Fox[FolderWithParent] = - Fox.successful(FolderWithParent(ObjectId(t._1), t._2, t._3.map(ObjectId(_)))) + private def parseMetadata(literal: String): Fox[JsArray] = + JsonHelper.parseAndValidateJson[JsArray](literal) override protected def readAccessQ(requestingUserId: ObjectId): SqlToken = readAccessQWithPrefix(requestingUserId, q"") @@ -246,6 +258,12 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) _ <- run(q"UPDATE webknossos.folders SET name = $name WHERE _id = $folderId".asUpdate) } yield () + def updateMetadata(folderId: ObjectId, metadata: JsArray)(implicit ctx: DBAccessContext): Fox[Unit] = + for { + _ <- assertUpdateAccess(folderId) + _ <- run(q"UPDATE webknossos.folders SET metadata = $metadata WHERE _id = $folderId".asUpdate) + } yield () + def findAllEditableIds(implicit ctx: DBAccessContext): Fox[List[ObjectId]] = for { updateAccessQuery <- accessQueryFromAccessQ(updateAccessQ) @@ -269,7 +287,7 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) for { accessQueryWithPrefix <- accessQueryFromAccessQWithPrefix(readAccessQWithPrefix, prefix = q"f.") accessQuery <- readAccessQuery - rows <- run(q"""SELECT f._id, f.name, fp._ancestor + rows <- run(q"""SELECT f._id, f.name, f.metadata, fp._ancestor FROM webknossos.folders_ f JOIN webknossos.folder_paths fp -- join to find immediate parent, this will also kick out self ON f._id = fp._descendant @@ -278,11 +296,11 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) FROM webknossos.folder_paths WHERE _ancestor = $folderId) AND $accessQueryWithPrefix - UNION ALL SELECT _id, name, NULL -- find self again, with no parent + UNION ALL SELECT _id, name, metadata, NULL -- find self again, with no parent FROM webknossos.folders_ WHERE _id = $folderId AND $accessQuery - """.as[(String, String, Option[String])]) + """.as[(String, String, String, Option[String])]) parsed <- Fox.combined(rows.toList.map(parseWithParent)) } yield parsed diff --git a/app/models/organization/OrganizationService.scala b/app/models/organization/OrganizationService.scala index 14586532162..98392061329 100644 --- a/app/models/organization/OrganizationService.scala +++ b/app/models/organization/OrganizationService.scala @@ -10,7 +10,7 @@ import models.dataset.{DataStore, DataStoreDAO} import models.folder.{Folder, FolderDAO, FolderService} import models.team.{PricingPlan, Team, TeamDAO} import models.user.{Invite, MultiUserDAO, User, UserDAO, UserService} -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.{JsArray, JsObject, Json} import utils.{ObjectId, WkConf} import scala.concurrent.{ExecutionContext, Future} @@ -111,7 +111,7 @@ class OrganizationService @Inject()(organizationDAO: OrganizationDAO, _ <- bool2Fox(existingOrganization.isEmpty) ?~> "organization.name.alreadyInUse" initialPricingParameters = if (conf.Features.isWkorgInstance) (PricingPlan.Basic, Some(3), Some(50000000000L)) else (PricingPlan.Custom, None, None) - organizationRootFolder = Folder(ObjectId.generate, folderService.defaultRootName) + organizationRootFolder = Folder(ObjectId.generate, folderService.defaultRootName, JsArray.empty) organization = Organization( ObjectId.generate, diff --git a/conf/evolutions/119-add-metadata-to-folders-and-datasets.sql b/conf/evolutions/119-add-metadata-to-folders-and-datasets.sql new file mode 100644 index 00000000000..08fdbda2e11 --- /dev/null +++ b/conf/evolutions/119-add-metadata-to-folders-and-datasets.sql @@ -0,0 +1,66 @@ +START TRANSACTION; + +do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 118, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql; + +DROP VIEW webknossos.folders_; +DROP VIEW webknossos.datasets_; + +-- Folder part +ALTER TABLE webknossos.folders ADD COLUMN metadata JSONB NOT NULL DEFAULT '[]'; +ALTER TABLE webknossos.folders ADD CONSTRAINT metadataIsJsonArray CHECK(jsonb_typeof(metadata) = 'array'); + +-- Dataset part +ALTER TABLE webknossos.datasets ADD COLUMN metadata JSONB NOT NULL DEFAULT '[]'; +ALTER TABLE webknossos.datasets ADD CONSTRAINT metadataIsJsonArray CHECK(jsonb_typeof(metadata) = 'array'); +-- Add existing details on species to metadata +UPDATE webknossos.datasets +SET metadata = CASE + WHEN details->>'species' IS NOT NULL THEN + metadata || jsonb_build_array( + jsonb_build_object( + 'type', 'string', + 'key', 'species', + 'value', details->>'species' + ) + ) + ELSE + metadata +END; +-- Add existing details on brain region to metadata +UPDATE webknossos.datasets +SET metadata = CASE + WHEN details->>'brainRegion' IS NOT NULL THEN + metadata || jsonb_build_array( + jsonb_build_object( + 'type', 'string', + 'key', 'brainRegion', + 'value', details->>'brainRegion' + ) + ) + ELSE + metadata +END; + +-- Add existing details on acquisition to metadata +UPDATE webknossos.datasets +SET metadata = CASE + WHEN details->>'acquisition' IS NOT NULL THEN + metadata || jsonb_build_array( + jsonb_build_object( + 'type', 'string', + 'key', 'acquisition', + 'value', details->>'acquisition' + ) + ) + ELSE + metadata +END; + +-- Drop details +ALTER TABLE webknossos.datasets DROP COLUMN details; + +CREATE VIEW webknossos.folders_ as SELECT * FROM webknossos.folders WHERE NOT isDeleted; +CREATE VIEW webknossos.datasets_ as SELECT * FROM webknossos.datasets WHERE NOT isDeleted; +UPDATE webknossos.releaseInformation SET schemaVersion = 119; + +COMMIT TRANSACTION; diff --git a/conf/evolutions/reversions/119-add-metadata-to-folders-and-datasets.sql b/conf/evolutions/reversions/119-add-metadata-to-folders-and-datasets.sql new file mode 100644 index 00000000000..cce83941f7d --- /dev/null +++ b/conf/evolutions/reversions/119-add-metadata-to-folders-and-datasets.sql @@ -0,0 +1,68 @@ +START TRANSACTION; + +do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 119, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql; + +DROP VIEW webknossos.folders_; +DROP VIEW webknossos.datasets_; + +-- Folder part +ALTER TABLE webknossos.folders DROP COLUMN metadata; + +-- Dataset part +ALTER TABLE webknossos.datasets ADD COLUMN details JSONB; +ALTER TABLE webknossos.datasets ADD CONSTRAINT detailsIsJsonObject CHECK(jsonb_typeof(details) = 'object'); + +-- Add existing info on species of metadata to details +UPDATE webknossos.datasets +SET details = jsonb_set(COALESCE(details, '{}'), '{species}', ( + SELECT to_jsonb(m.value) + FROM jsonb_to_recordset(metadata) AS m(key text, value text) + WHERE m.key = 'species' + LIMIT 1 +)) +WHERE EXISTS ( + SELECT 1 + FROM jsonb_array_elements(metadata) AS m + WHERE m->>'key' = 'species' +); + + +-- Add existing info on species of brainRegion to details +UPDATE webknossos.datasets +SET details = jsonb_set(COALESCE(details, '{}'), '{brainRegion}', ( + SELECT to_jsonb(m.value) + FROM jsonb_to_recordset(metadata) AS m(key text, value text) + WHERE m.key = 'brainRegion' + LIMIT 1 +)) +WHERE EXISTS ( + SELECT 1 + FROM jsonb_array_elements(metadata) AS m + WHERE m->>'key' = 'brainRegion' +); + + +-- Add existing info on species of acquisition to details +UPDATE webknossos.datasets +SET details = jsonb_set(COALESCE(details, '{}'), '{acquisition}', ( + SELECT to_jsonb(m.value) + FROM jsonb_to_recordset(metadata) AS m(key text, value text) + WHERE m.key = 'acquisition' + LIMIT 1 +)) +WHERE EXISTS ( + SELECT 1 + FROM jsonb_array_elements(metadata) AS m + WHERE m->>'key' = 'acquisition' +); + + + +-- Drop details +ALTER TABLE webknossos.datasets DROP COLUMN metadata; + +CREATE VIEW webknossos.folders_ as SELECT * FROM webknossos.folders WHERE NOT isDeleted; +CREATE VIEW webknossos.datasets_ as SELECT * FROM webknossos.datasets WHERE NOT isDeleted; +UPDATE webknossos.releaseInformation SET schemaVersion = 118; + +COMMIT TRANSACTION; diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 37a528d5ad5..77273ddff2f 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -1170,6 +1170,7 @@ export type DatasetUpdater = { isPublic?: boolean; tags?: string[]; folderId?: string; + metadata?: APIDataset["metadata"]; }; export function updateDatasetPartial( diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index 67c250fa955..be3c4d30dbc 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -41,7 +41,7 @@ import classNames from "classnames"; import { EmptyObject } from "types/globals"; type FolderItemWithName = FolderItem & { name: string }; -type DatasetOrFolder = APIDatasetCompact | FolderItemWithName; +export type DatasetOrFolder = APIDatasetCompact | FolderItemWithName; type RowRenderer = DatasetRenderer | FolderRenderer; const { ThinSpace } = Unicode; @@ -392,7 +392,7 @@ class FolderRenderer { class DatasetTable extends React.PureComponent { state: State = { sortedInfo: { - columnKey: useLruRank ? "" : "created", + columnKey: useLruRank ? undefined : "created", order: "descend", }, prevSearchQuery: "", diff --git a/frontend/javascripts/dashboard/dataset/dataset_collection_context.tsx b/frontend/javascripts/dashboard/dataset/dataset_collection_context.tsx index 95905f9f02d..3c8bf0a80b1 100644 --- a/frontend/javascripts/dashboard/dataset/dataset_collection_context.tsx +++ b/frontend/javascripts/dashboard/dataset/dataset_collection_context.tsx @@ -4,6 +4,7 @@ import type { APIDatasetCompact, APIDatasetCompactWithoutStatusAndLayerNames, FolderItem, + APIDataset, } from "types/api_flow_types"; import { DatasetUpdater, getDatastores, triggerDatasetCheck } from "admin/admin_rest_api"; import UserLocalStorage from "libs/user_local_storage"; @@ -32,7 +33,7 @@ export type DatasetCollectionContextValue = { datasetId: APIDatasetId, datasetsToUpdate?: Array, ) => Promise; - updateCachedDataset: (id: APIDatasetId, updater: DatasetUpdater) => Promise; + updateCachedDataset: (id: APIDatasetId, updater: DatasetUpdater) => Promise; activeFolderId: string | null; setActiveFolderId: (id: string | null) => void; mostRecentlyUsedActiveFolderId: string | null; @@ -160,7 +161,7 @@ export default function DatasetCollectionContextProvider({ } async function updateCachedDataset(id: APIDatasetId, updater: DatasetUpdater) { - await updateDatasetMutation.mutateAsync([id, updater]); + return await updateDatasetMutation.mutateAsync([id, updater]); } const getBreadcrumbs = (dataset: APIDatasetCompactWithoutStatusAndLayerNames) => { diff --git a/frontend/javascripts/dashboard/dataset/queries.tsx b/frontend/javascripts/dashboard/dataset/queries.tsx index 718b8e8a4c6..78ef64bd766 100644 --- a/frontend/javascripts/dashboard/dataset/queries.tsx +++ b/frontend/javascripts/dashboard/dataset/queries.tsx @@ -279,7 +279,7 @@ export function useCreateFolderMutation() { queryClient.setQueryData( mutationKey, transformHierarchy((oldItems: FlatFolderTreeItem[] | undefined) => - (oldItems || []).concat([{ ...newFolder, parent: parentId }]), + (oldItems || []).concat([{ ...newFolder, parent: parentId, metadata: [] }]), ), ); }, @@ -421,6 +421,12 @@ export function useUpdateDatasetMutation(folderId: string | null) { }) .filter((dataset: APIDatasetCompact) => dataset.folderId === folderId), ); + const updatedDatasetId = { + name: updatedDataset.name, + owningOrganization: updatedDataset.owningOrganization, + }; + // Also update the cached dataset under the key "datasetById". + queryClient.setQueryData(["datasetById", updatedDatasetId], updatedDataset); const targetFolderId = updatedDataset.folderId; if (targetFolderId !== folderId) { // The dataset was moved to another folder. Add the dataset to that target folder @@ -585,6 +591,7 @@ export function getFolderHierarchy(folderTree: FlatFolderTreeItem[]): FolderHier title: folderTreeItem.name, isEditable: folderTreeItem.isEditable, parent: folderTreeItem.parent, + metadata: folderTreeItem.metadata, children: [], }; if (folderTreeItem.parent == null) { diff --git a/frontend/javascripts/dashboard/folders/details_sidebar.tsx b/frontend/javascripts/dashboard/folders/details_sidebar.tsx index 4abfeee27e8..4f035f431ba 100644 --- a/frontend/javascripts/dashboard/folders/details_sidebar.tsx +++ b/frontend/javascripts/dashboard/folders/details_sidebar.tsx @@ -23,6 +23,7 @@ import { useSelector } from "react-redux"; import { OxalisState } from "oxalis/store"; import { getOrganization } from "admin/admin_rest_api"; import { useQuery } from "@tanstack/react-query"; +import MetadataTable from "./metadata_table"; import Markdown from "libs/markdown_adapter"; export function DetailsSidebar({ @@ -89,7 +90,12 @@ function getMaybeSelectMessage(datasetCount: number) { function DatasetDetails({ selectedDataset }: { selectedDataset: APIDatasetCompact }) { const context = useDatasetCollectionContext(); - const { data: fullDataset, isFetching } = useDatasetQuery(selectedDataset); + // exactDatasetId is needed to prevent refetching when some dataset property of selectedDataset was changed. + const exactDatasetId = { + owningOrganization: selectedDataset.owningOrganization, + name: selectedDataset.name, + }; + const { data: fullDataset, isFetching } = useDatasetQuery(exactDatasetId); const activeUser = useSelector((state: OxalisState) => state.activeUser); const { data: owningOrganization } = useQuery( ["organizations", selectedDataset.owningOrganization], @@ -173,14 +179,22 @@ function DatasetDetails({ selectedDataset }: { selectedDataset: APIDatasetCompac )} - - {selectedDataset.isActive ? ( -
-
Tags
- -
- ) : null} + {selectedDataset.isActive ? ( +
+
Tags
+ +
+ ) : null} + + {fullDataset && ( + /* The key is crucial to enforce rerendering when the dataset changes. This is necessary for the MetadataTable to work correctly. */ + + )} + {fullDataset?.usedStorageBytes && fullDataset.usedStorageBytes > 10000 ? (
@@ -281,7 +295,11 @@ function FolderDetails({ . {message}

Access Permissions
- +
+ +
+ {/* The key is crucial to enforce rerendering when the folder changes. This is necessary for the MetadataTable to work correctly. */} +
) : error ? ( "Could not load folder." diff --git a/frontend/javascripts/dashboard/folders/folder_tree.tsx b/frontend/javascripts/dashboard/folders/folder_tree.tsx index d7df3d48d77..bcbba1886b5 100644 --- a/frontend/javascripts/dashboard/folders/folder_tree.tsx +++ b/frontend/javascripts/dashboard/folders/folder_tree.tsx @@ -88,6 +88,7 @@ export function FolderTreeSidebar({ const doesEventReferToTreeUi = event.nativeEvent.target.closest(".ant-tree") != null; if (keys.length > 0 && doesEventReferToTreeUi) { context.setActiveFolderId(keys[0] as string); + context.setSelectedDatasets([]); } }, [context], diff --git a/frontend/javascripts/dashboard/folders/metadata_table.tsx b/frontend/javascripts/dashboard/folders/metadata_table.tsx new file mode 100644 index 00000000000..b2fc7d2b04c --- /dev/null +++ b/frontend/javascripts/dashboard/folders/metadata_table.tsx @@ -0,0 +1,468 @@ +import { + DeleteOutlined, + FieldNumberOutlined, + FieldStringOutlined, + PlusOutlined, + TagsOutlined, +} from "@ant-design/icons"; +import { + MenuProps, + InputNumberProps, + InputNumber, + Input, + Select, + Typography, + Dropdown, + Button, +} from "antd"; +import { + DatasetCollectionContextValue, + useDatasetCollectionContext, +} from "dashboard/dataset/dataset_collection_context"; +import { useIsMounted, useStateWithRef } from "libs/react_hooks"; +import Toast from "libs/toast"; +import _ from "lodash"; +import React, { useEffect } from "react"; +import { useState } from "react"; +import { APIDataset, Folder, APIMetadata, APIMetadataEnum } from "types/api_flow_types"; + +type APIMetadataWithError = APIMetadata & { error?: string | null }; +type IndexedMetadataEntries = APIMetadataWithError[]; + +function getMetadataTypeLabel(type: APIMetadata["type"]) { + switch (type) { + case "string": + return ( + + Text + + ); + case "number": + return ( + + Number + + ); + case "string[]": + return ( + + Multi-Item Text + + ); + } +} + +type EmptyMetadataPlaceholderProps = { + addNewEntryMenuItems: MenuProps; +}; +const EmptyMetadataPlaceholder: React.FC = ({ + addNewEntryMenuItems, +}) => { + return ( +
+ Metadata preview + + + + + +
+ ); +}; + +interface MetadataValueInputProps { + record: APIMetadataWithError; + index: number; + focusedRow: number | null; + setFocusedRow: (row: number | null) => void; + updateMetadataValue: (index: number, newValue: number | string | string[]) => void; + isSaving: boolean; + availableStrArrayTagOptions: { value: string; label: string }[]; +} + +const MetadataValueInput: React.FC = ({ + record, + index, + focusedRow, + setFocusedRow, + updateMetadataValue, + isSaving, + availableStrArrayTagOptions, +}) => { + const isFocused = index === focusedRow; + const sharedProps = { + className: isFocused ? undefined : "transparent-input", + onFocus: () => setFocusedRow(index), + onBlur: () => setFocusedRow(null), + placeholder: "Value", + size: "small" as InputNumberProps["size"], + disabled: isSaving, + }; + + switch (record.type) { + case APIMetadataEnum.NUMBER: + return ( + updateMetadataValue(index, newNum || 0)} + {...sharedProps} + /> + ); + case APIMetadataEnum.STRING: + return ( + updateMetadataValue(index, evt.target.value)} + {...sharedProps} + /> + ); + case APIMetadataEnum.STRING_ARRAY: + return ( + setFocusedRow(index)} + onBlur={() => setFocusedRow(null)} + value={record.key} + onChange={(evt) => updateMetadataKey(index, evt.target.value)} + placeholder="Property" + size="small" + disabled={isSaving} + id={getKeyInputIdForIndex(index)} + /> + {record.error != null ? ( + <> +
+ + {record.error} + + + ) : null} + + ); + }; + + const getDeleteEntryButton = (_: APIMetadataWithError, index: number) => ( +
+
+ ); + + const addNewEntryMenuItems = getTypeSelectDropdownMenu(); + + return ( +
+
Metadata
+
+ {/* Not using AntD Table to have more control over the styling. */} + {metadata.length > 0 ? ( + + {/* Each row except the last row has a custom horizontal divider created via a css pseudo element. */} + + + + + + + + {metadata.map((record, index) => ( + + + + + + + ))} + + + + +
Property + Value +
{getKeyInput(record, index)}: + + {getDeleteEntryButton(record, index)}
+
+ + + +
+
+ ) : ( + + )} +
+
+ ); +} diff --git a/frontend/javascripts/dashboard/publication_card.tsx b/frontend/javascripts/dashboard/publication_card.tsx index 01028d74202..108612f0e17 100644 --- a/frontend/javascripts/dashboard/publication_card.tsx +++ b/frontend/javascripts/dashboard/publication_card.tsx @@ -4,12 +4,7 @@ import Markdown from "libs/markdown_adapter"; import React, { useState } from "react"; import classNames from "classnames"; import { Link } from "react-router-dom"; -import type { - APIDataset, - APIDatasetDetails, - APIPublication, - APIPublicationAnnotation, -} from "types/api_flow_types"; +import type { APIDataset, APIPublication, APIPublicationAnnotation } from "types/api_flow_types"; import { formatScale } from "libs/format_utils"; import { getThumbnailURL, @@ -18,7 +13,14 @@ import { getDatasetExtentAsString, } from "oxalis/model/accessors/dataset_accessor"; import { compareBy } from "libs/utils"; -type ExtendedDatasetDetails = APIDatasetDetails & { + +type DatasetDetails = { + species?: string; + brainRegion?: string; + acquisition?: string; +}; + +type ExtendedDatasetDetails = DatasetDetails & { name: string; scale: string; extent: string; @@ -49,8 +51,14 @@ function getDisplayName(item: PublicationItem): string { : item.dataset.displayName; } -function getDetails(item: PublicationItem): ExtendedDatasetDetails { - const { dataSource, details } = item.dataset; +function getExtendedDetails(item: PublicationItem): ExtendedDatasetDetails { + const { dataSource, metadata } = item.dataset; + const details = {} as DatasetDetails; + metadata?.forEach((entry) => { + if (entry.key === "species" || entry.key === "brainRegion" || entry.key === "acquisition") { + details[entry.key] = entry.value.toString(); + } + }); return { ...details, scale: formatScale(dataSource.scale, 0), @@ -264,7 +272,7 @@ function PublicationThumbnail({ const segmentationThumbnailURL = hasSegmentation(activeItem.dataset) ? getSegmentationThumbnailURL(activeItem.dataset) : null; - const details = getDetails(activeItem); + const extendedDetails = getExtendedDetails(activeItem); return (
@@ -293,7 +301,7 @@ function PublicationThumbnail({ }} /> )} - + {sortedItems.length > 1 && ( model.description, (model) => model.title, (model) => - model.datasets.flatMap((dataset) => [dataset.name, dataset.description, dataset.details]), + model.datasets.flatMap((dataset) => [dataset.name, dataset.description, dataset.metadata]), ], props.searchQuery, ).sort(Utils.compareBy((publication) => publication.publicationDate, false)); diff --git a/frontend/javascripts/libs/react_hooks.ts b/frontend/javascripts/libs/react_hooks.ts index 12ed35e27cf..d9f0d9aa462 100644 --- a/frontend/javascripts/libs/react_hooks.ts +++ b/frontend/javascripts/libs/react_hooks.ts @@ -1,5 +1,5 @@ import constants from "oxalis/constants"; -import { useState, useEffect, useRef } from "react"; +import { useState, useEffect, useRef, useCallback } from "react"; import { useLocation } from "react-router-dom"; import { KEYBOARD_BUTTON_LOOP_INTERVAL } from "./input"; @@ -200,3 +200,37 @@ export function useEffectOnlyOnce(callback: () => void | (() => void)) { return callback(); }, []); } + +// This hook allows to access the current state value in an asynchronous function call. +// Due to the nature of hooks, the ref value might be one render cycle ahead of the state value. +// If the ref value should preferably be one render cycle behind the state value, +// use a different hook that uses an effect instead of a wrapped state setter. +export function useStateWithRef(initialValue: T) { + const [state, setState] = useState(initialValue); + const ref = useRef(state); + + const wrappedSetState = (newState: T | ((prevState: T) => T)) => { + setState((prevState: T) => { + const nextState = + typeof newState === "function" ? (newState as (prevState: T) => T)(prevState) : newState; + ref.current = nextState; + return nextState; + }); + }; + + return [state, ref, wrappedSetState] as const; +} + +export function useIsMounted() { + const isMountedRef = useRef(true); + + useEffect(() => { + return () => { + isMountedRef.current = false; + }; + }, []); + + const isMounted = useCallback(() => isMountedRef.current, []); + + return isMounted; +} diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts index 7019609dd72..8231c85a077 100644 --- a/frontend/javascripts/oxalis/default_state.ts +++ b/frontend/javascripts/oxalis/default_state.ts @@ -129,7 +129,7 @@ const defaultState: OxalisState = { team: "", }, }, - details: null, + metadata: null, tags: [], isPublic: false, isActive: true, diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/bounding_box_tab.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/bounding_box_tab.tsx index d0c5ff2b264..32d0ac8b3d7 100644 --- a/frontend/javascripts/oxalis/view/right-border-tabs/bounding_box_tab.tsx +++ b/frontend/javascripts/oxalis/view/right-border-tabs/bounding_box_tab.tsx @@ -106,12 +106,11 @@ export default function BoundingBoxTab() { APIJobType.EXPORT_TIFF, ); - // biome-ignore lint/correctness/useExhaustiveDependencies: Always try to scroll the active bounding box into view. useEffect(() => { if (bboxTableRef.current != null && activeBoundingBoxId != null) { bboxTableRef.current.scrollTo({ key: activeBoundingBoxId }); } - }, [activeBoundingBoxId, bboxTableRef.current]); + }, [activeBoundingBoxId]); const boundingBoxWrapperTableColumns = [ { diff --git a/frontend/javascripts/test/backend-snapshot-tests/folders.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/folders.e2e.ts index e5201d5ca5f..f68958c9820 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/folders.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/folders.e2e.ts @@ -10,6 +10,7 @@ import { import Request from "libs/request"; import * as foldersApi from "admin/api/folders"; import test from "ava"; +import { APIMetadataEnum } from "types/api_flow_types"; test.before("Reset database and change token", async () => { resetDatabase(); setCurrToken(tokenUserA); @@ -42,6 +43,7 @@ test("updateFolder", async (t) => { id: organizationXRootFolderId, allowedTeams: [], name: newName, + metadata: [], }); t.is(updatedFolder.name, newName); @@ -69,6 +71,7 @@ test("addAllowedTeamToFolder", async (t) => { id: subFolderId, allowedTeams: [teamId], name: "A subfolder!", + metadata: [{ type: APIMetadataEnum.STRING, key: "foo", value: "bar" }], }); t.snapshot(updatedFolderWithTeam, { diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md index c57f3de9e99..ffc26e90cbd 100644 --- a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md +++ b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md @@ -283,7 +283,6 @@ Generated by [AVA](https://avajs.dev). url: 'http://localhost:9000', }, description: null, - details: null, displayName: null, folderId: '570b9f4e4bb848d0885ea917', isActive: true, @@ -292,6 +291,13 @@ Generated by [AVA](https://avajs.dev). isUnreported: false, lastUsedByUser: 0, logoUrl: '/assets/images/mpi-logos.svg', + metadata: [ + { + key: 'key', + type: 'number', + value: 4, + }, + ], name: 'confocal-multi_knossos', owningOrganization: 'Organization_X', publication: null, @@ -425,7 +431,6 @@ Generated by [AVA](https://avajs.dev). url: 'http://localhost:9000', }, description: null, - details: null, displayName: null, folderId: '570b9f4e4bb848d0885ea917', isActive: true, @@ -434,6 +439,7 @@ Generated by [AVA](https://avajs.dev). isUnreported: false, lastUsedByUser: 0, logoUrl: '/assets/images/mpi-logos.svg', + metadata: [], name: 'l4_sample', owningOrganization: 'Organization_X', publication: null, diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap index c1bcfcfb298..8e78e8ecd38 100644 Binary files a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap and b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap differ diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.md b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.md index adf2b9863e9..8b52dfa7d45 100644 --- a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.md +++ b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.md @@ -10,18 +10,27 @@ Generated by [AVA](https://avajs.dev). { id: '570b9f4e4bb848d08880712a', isEditable: true, + metadata: [], name: 'A subfolder!', parent: '570b9f4e4bb848d0885ea917', }, { id: '570b9f4e4bb848d08880712b', isEditable: true, + metadata: [], name: 'Another subfolder!', parent: '570b9f4e4bb848d0885ea917', }, { id: '570b9f4e4bb848d0885ea917', isEditable: true, + metadata: [ + { + key: 'key', + type: 'number', + value: 10, + }, + ], name: 'Organization_X', parent: null, }, @@ -34,6 +43,13 @@ Generated by [AVA](https://avajs.dev). allowedTeamsCumulative: [], id: '570b9f4e4bb848d0885ea917', isEditable: true, + metadata: [ + { + key: 'key', + type: 'number', + value: 10, + }, + ], name: 'Organization_X', } @@ -44,6 +60,7 @@ Generated by [AVA](https://avajs.dev). allowedTeamsCumulative: [], id: '570b9f4e4bb848d0885ea917', isEditable: true, + metadata: [], name: 'renamed organization x root folder', } @@ -54,6 +71,7 @@ Generated by [AVA](https://avajs.dev). allowedTeamsCumulative: [], id: 'id', isEditable: true, + metadata: [], name: 'a newly created folder!', } @@ -76,6 +94,13 @@ Generated by [AVA](https://avajs.dev). ], id: '570b9f4e4bb848d08880712a', isEditable: true, + metadata: [ + { + key: 'foo', + type: 'string', + value: 'bar', + }, + ], name: 'A subfolder!', } @@ -98,5 +123,12 @@ Generated by [AVA](https://avajs.dev). ], id: '570b9f4e4bb848d08880712a', isEditable: false, + metadata: [ + { + key: 'foo', + type: 'string', + value: 'bar', + }, + ], name: 'A subfolder!', } diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.snap b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.snap index 425878c1cf9..94a25562638 100644 Binary files a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.snap and b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/folders.e2e.js.snap differ diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.snap b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.snap index 03f50e3ed35..6c010b6c128 100644 Binary files a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.snap and b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.snap differ diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index c7f087825a2..d56298388ea 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -24,7 +24,7 @@ import type { UnitLong, } from "oxalis/constants"; import { PricingPlanEnum } from "admin/organization/pricing_plan_utils"; -import { EmptyObject } from "./globals"; +import { EmptyObject, ValueOf } from "./globals"; export type AdditionalCoordinate = { name: string; value: number }; @@ -171,11 +171,19 @@ export function areDatasetsIdentical(a: APIDatasetId, b: APIDatasetId) { return a.owningOrganization === b.owningOrganization && a.name === b.name; } export type APIDatasetId = Readonly; -export type APIDatasetDetails = { - readonly species?: string; - readonly brainRegion?: string; - readonly acquisition?: string; + +export enum APIMetadataEnum { + STRING = "string", + NUMBER = "number", + STRING_ARRAY = "string[]", +} +type APIMetadataType = ValueOf; +export type APIMetadata = { + type: APIMetadataType; + key: string; + value: string | number | string[]; }; +export type APIMetadataEntries = APIMetadata[]; type MutableAPIDatasetBase = MutableAPIDatasetId & { isUnreported: boolean; @@ -185,7 +193,7 @@ type MutableAPIDatasetBase = MutableAPIDatasetId & { created: number; dataStore: APIDataStore; description: string | null | undefined; - details: APIDatasetDetails | null | undefined; + metadata: APIMetadataEntries | null | undefined; isEditable: boolean; isPublic: boolean; displayName: string | null | undefined; @@ -237,7 +245,7 @@ export type APIDatasetCompact = APIDatasetCompactWithoutStatusAndLayerNames & { }; export function convertDatasetToCompact(dataset: APIDataset): APIDatasetCompact { - const [colorLayerNames, segmentationLayerNames] = _.partition( + const [segmentationLayerNames, colorLayerNames] = _.partition( dataset.dataSource.dataLayers, (layer) => layer.category === "segmentation", ).map((layers) => layers.map((layer) => layer.name).sort()); @@ -1068,21 +1076,23 @@ export type VoxelyticsLogLine = { wk_url: string; }; -// Backend type +// Backend type returned by the getFolderTree api method. export type FlatFolderTreeItem = { name: string; id: string; parent: string | null; + metadata: APIMetadataEntries; isEditable: boolean; }; -// Frontend type +// Frontend type of FlatFolderTreeItem with inferred nested structure. export type FolderItem = { title: string; key: string; // folder id parent: string | null | undefined; children: FolderItem[]; isEditable: boolean; + metadata: APIMetadataEntries; // Can be set so that the antd tree component can disable // individual folder items. disabled?: boolean; @@ -1093,6 +1103,7 @@ export type Folder = { id: string; allowedTeams: APITeam[]; allowedTeamsCumulative: APITeam[]; + metadata: APIMetadataEntries; isEditable: boolean; }; @@ -1100,6 +1111,7 @@ export type FolderUpdater = { id: string; name: string; allowedTeams: string[]; + metadata: APIMetadataEntries; }; export enum CAMERA_POSITIONS { diff --git a/frontend/stylesheets/_dashboard.less b/frontend/stylesheets/_dashboard.less index 928bb1e1b74..880f4abc5fd 100644 --- a/frontend/stylesheets/_dashboard.less +++ b/frontend/stylesheets/_dashboard.less @@ -212,7 +212,7 @@ pre.dataset-import-folder-structure-hint { font-size: 16px; font-weight: 500; display: block; - min-height: 30px + min-height: 30px; } .dataset-table-name-container { @@ -220,3 +220,95 @@ pre.dataset-import-folder-structure-hint { vertical-align: middle; max-width: 520px; } + +.top-aligned-column { + vertical-align: top; +} + +.metadata-table-wrapper { + border: var(--ant-line-width) var(--ant-line-type) var(--ant-color-border); + color: var(--ant-tag-default-color); + background: var(--ant-tag-default-bg); + border-radius: var(--ant-border-radius-sm); + padding: 2px; + .metadata-table { + color: var(--ant-tag-default-color); + background: var(--ant-tag-default-bg); + border-collapse: collapse; /* Ensure border-collapse is set to separate */ + table-layout: fixed; + + tr td { + padding-top: 4px; + padding-bottom: 4px; + } + tbody tr td:last-child { + padding: 0px; + } + + tr { + // A pseudo element is created to add a horizontal line between the rows. It does not stretch the entire width of the table. + position: relative; + &:not(:last-child)::after { + content: ""; + position: absolute; + left: 8px; + right: 8px; + top: 0; + height: var(--ant-line-width); + background-color: var(--ant-color-border); + } + } + + .transparent-input { + background: transparent; + border-color: transparent; + .ant-select-selector { + border-color: transparent; + } + } + + th { + padding: 4px !important; + padding-left: 9px !important; + font-weight: normal; + text-align: left; + } + .ant-table-cell { + padding: 4px 4px 4px 0px !important; + } + + thead tr th:first-child, + tbody tr td:first-child, + thead tr th:nth-child(3), + tbody tr td:nth-child(3) { + .ant-input-number, + .ant-select, + input { + width: 114.5px; + } + // Due to the border around the input elements, the width of the td has to be increased by 2px. + width: 116.5px; + } + tbody tr td:first-child, + tbody tr td:nth-child(3) { + vertical-align: top; + } + thead tr th:nth-child(2), + tbody tr td:nth-child(2) { + width: 5px; + // Center the : + text-align: center; + vertical-align: top; + } + thead tr th:nth-child(4), + tbody tr td:nth-child(4) { + width: 20px; + vertical-align: top; + } + } + + .empty-metadata-placeholder { + padding: 16px; + text-align: center; + } +} diff --git a/public/images/metadata-teaser.svg b/public/images/metadata-teaser.svg new file mode 100644 index 00000000000..a27a2b12479 --- /dev/null +++ b/public/images/metadata-teaser.svg @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/db/dataSets.csv b/test/db/dataSets.csv index 11b23b3bc6c..5e8dc9010e3 100644 --- a/test/db/dataSets.csv +++ b/test/db/dataSets.csv @@ -1,7 +1,7 @@ -_id,_dataStore,_organization,_publication,_uploader,_folder,inboxSourceHash,defaultViewConfiguration,adminViewConfiguration,description,displayName,isPublic,isUsable,name,voxelSizeFactor,voxelSizeUnit,status,sharingToken,logoUrl,sortingKey,details,tags,created,isDeleted -'570b9f4e4bb848d0885ee711','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'2012-06-28_Cortex',,,'No longer available on datastore.',,,'2016-04-11T12:57:50.082Z',,{},'2016-04-11T12:57:50.082Z',f -'570b9f4e4bb848d0885ee712','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'Experiment_001',,,'No longer available on datastore.',,,'2016-04-11T12:57:50.079Z',,{},'2016-04-11T12:57:50.079Z',f -'570b9f4e4bb848d0885ee713','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'2012-09-28_ex145_07x2',,,'No longer available on datastore.',,,'2016-04-11T12:57:50.080Z',,{},'2016-04-11T12:57:50.080Z',f -'570b9fd34bb848d0885ee716','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'rgb',,,'No longer available on datastore.',,,'2016-04-11T13:00:03.792Z',,{},'2016-04-11T13:00:03.792Z',f -'59e9cfbdba632ac2ab8b23b3','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,t,'confocal-multi_knossos','(22,22,44.599998474121094)','nanometer','',,,'2017-10-20T10:28:13.763Z',,{},'2017-10-20T10:28:13.763Z',f -'59e9cfbdba632ac2ab8b23b5','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,t,'l4_sample','(11.239999771118164,11.239999771118164,28)','nanometer','',,,'2017-10-20T10:28:13.789Z',,{},'2017-10-20T10:28:13.789Z',f +_id,_dataStore,_organization,_publication,_uploader,_folder,inboxSourceHash,defaultViewConfiguration,adminViewConfiguration,description,displayName,isPublic,isUsable,name,voxelSizeFactor,voxelSizeUnit,status,sharingToken,logoUrl,sortingKey,metadata,tags,created,isDeleted +'570b9f4e4bb848d0885ee711','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'2012-06-28_Cortex',,,'No longer available on datastore.',,,'2016-04-11T12:57:50.082Z','[]',{},'2016-04-11T12:57:50.082Z',f +'570b9f4e4bb848d0885ee712','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'Experiment_001',,,'No longer available on datastore.',,,'2016-04-11T12:57:50.079Z','[]',{},'2016-04-11T12:57:50.079Z',f +'570b9f4e4bb848d0885ee713','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'2012-09-28_ex145_07x2',,,'No longer available on datastore.',,,'2016-04-11T12:57:50.080Z','[]',{},'2016-04-11T12:57:50.080Z',f +'570b9fd34bb848d0885ee716','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,f,'rgb',,,'No longer available on datastore.',,,'2016-04-11T13:00:03.792Z',[],{},'2016-04-11T13:00:03.792Z',f +'59e9cfbdba632ac2ab8b23b3','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,t,'confocal-multi_knossos','(22,22,44.599998474121094)','nanometer','',,,'2017-10-20T10:28:13.763Z','[{"key": "key","type": "number","value": 4}]',{},'2017-10-20T10:28:13.763Z',f +'59e9cfbdba632ac2ab8b23b5','localhost','5ab0c6a674d0af7b003b23ac',,,'570b9f4e4bb848d0885ea917',,,,,,f,t,'l4_sample','(11.239999771118164,11.239999771118164,28)','nanometer','',,,'2017-10-20T10:28:13.789Z','[]',{},'2017-10-20T10:28:13.789Z',f diff --git a/test/db/folders.csv b/test/db/folders.csv index 82c1fbb9619..b1f2ad38c53 100644 --- a/test/db/folders.csv +++ b/test/db/folders.csv @@ -1,5 +1,5 @@ -_id,name,isDeleted -'570b9f4e4bb848d0885ea917','Organization_X',f -'570b9f4e4bb848d08880712a','A subfolder!',f -'570b9f4e4bb848d08880712b','Another subfolder!',f -'570b9f4e4bb848d088a83aef','Organization_Y',f +_id,name,isDeleted,metadata +'570b9f4e4bb848d0885ea917','Organization_X',f,'[{"key": "key","type": "number","value": 10}]' +'570b9f4e4bb848d08880712a','A subfolder!',f,'[]' +'570b9f4e4bb848d08880712b','Another subfolder!',f,'[]' +'570b9f4e4bb848d088a83aef','Organization_Y',f,'[]' diff --git a/tools/postgres/schema.sql b/tools/postgres/schema.sql index d14929480f0..389aa702a53 100644 --- a/tools/postgres/schema.sql +++ b/tools/postgres/schema.sql @@ -20,7 +20,7 @@ CREATE TABLE webknossos.releaseInformation ( schemaVersion BIGINT NOT NULL ); -INSERT INTO webknossos.releaseInformation(schemaVersion) values(118); +INSERT INTO webknossos.releaseInformation(schemaVersion) values(119); COMMIT TRANSACTION; @@ -123,14 +123,14 @@ CREATE TABLE webknossos.datasets( sharingToken CHAR(256), logoUrl VARCHAR(2048), sortingKey TIMESTAMPTZ NOT NULL, - details JSONB, + metadata JSONB NOT NULL DEFAULT '[]', tags VARCHAR(256)[] NOT NULL DEFAULT '{}', created TIMESTAMPTZ NOT NULL DEFAULT NOW(), isDeleted BOOLEAN NOT NULL DEFAULT false, UNIQUE (name, _organization), CONSTRAINT defaultViewConfigurationIsJsonObject CHECK(jsonb_typeof(defaultViewConfiguration) = 'object'), CONSTRAINT adminViewConfigurationIsJsonObject CHECK(jsonb_typeof(adminViewConfiguration) = 'object'), - CONSTRAINT detailsIsJsonObject CHECK(jsonb_typeof(details) = 'object') + CONSTRAINT metadataIsJsonArray CHECK(jsonb_typeof(metadata) = 'array') ); CREATE TYPE webknossos.DATASET_LAYER_CATEGORY AS ENUM ('color', 'mask', 'segmentation'); @@ -517,7 +517,9 @@ CREATE TABLE webknossos.credentials( CREATE TABLE webknossos.folders( _id CHAR(24) PRIMARY KEY, name TEXT NOT NULL CHECK (name !~ '/'), - isDeleted BOOLEAN NOT NULL DEFAULT false + isDeleted BOOLEAN NOT NULL DEFAULT false, + metadata JSONB NOT NULL DEFAULT '[]', + CONSTRAINT metadataIsJsonArray CHECK(jsonb_typeof(metadata) = 'array') ); CREATE TABLE webknossos.folder_paths(