From 45c88c3b8034221b3a3e694a179b477f17a105b4 Mon Sep 17 00:00:00 2001 From: Ben Spencer Date: Sun, 23 May 2021 16:01:03 +0100 Subject: [PATCH 1/7] CE3: Shared base class for law testing suites. --- .../io/catbird/util/effect/BaseLawSuite.scala | 13 +++++++++++++ .../io/catbird/util/effect/FutureSuite.scala | 12 ++---------- .../io/catbird/util/effect/RerunnableSuite.scala | 16 +++------------- 3 files changed, 18 insertions(+), 23 deletions(-) create mode 100644 effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala diff --git a/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala new file mode 100644 index 00000000..ac3c1b48 --- /dev/null +++ b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala @@ -0,0 +1,13 @@ +package io.catbird.util +package effect + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.prop.Configuration +import org.typelevel.discipline.scalatest.FunSuiteDiscipline + +abstract class BaseLawSuite + extends AnyFunSuite + with FunSuiteDiscipline + with Configuration + with ArbitraryInstances + with EqInstances diff --git a/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala index 8a61fa22..1023bd6a 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala @@ -7,17 +7,9 @@ import cats.laws.discipline.MonadErrorTests import cats.laws.discipline.arbitrary._ import com.twitter.conversions.DurationOps._ import com.twitter.util.Future -import io.catbird.util.{ ArbitraryInstances, EqInstances, futureEqWithFailure } -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.prop.Configuration -import org.typelevel.discipline.scalatest.FunSuiteDiscipline +import io.catbird.util.futureEqWithFailure -class FutureSuite - extends AnyFunSuite - with FunSuiteDiscipline - with Configuration - with ArbitraryInstances - with EqInstances { +class FutureSuite extends BaseLawSuite { implicit def futureEq[A](implicit A: Eq[A]): Eq[Future[A]] = futureEqWithFailure(1.seconds) diff --git a/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala index e08deb91..fd624f3b 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala @@ -9,19 +9,9 @@ import cats.instances.tuple._ import cats.instances.unit._ import cats.laws.discipline.arbitrary._ import com.twitter.util.{ Await, Monitor, Throw } -import io.catbird.util.{ ArbitraryInstances, EqInstances, Rerunnable } -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.prop.Configuration -import org.typelevel.discipline.scalatest.FunSuiteDiscipline - -class RerunnableSuite - extends AnyFunSuite - with FunSuiteDiscipline - with Configuration - with ArbitraryInstances - with SyncTypeGenerators - with EqInstances - with Runners { +import io.catbird.util.Rerunnable + +class RerunnableSuite extends BaseLawSuite with SyncTypeGenerators with Runners { // This includes tests for Clock, MonadCancel, and MonadError checkAll("Rerunnable[Int]", SyncTests[Rerunnable].sync[Int, Int, Int]) From 277bcd9cb9200c5a582a7983c732c9361386261d Mon Sep 17 00:00:00 2001 From: Ben Spencer Date: Sun, 23 May 2021 15:35:40 +0100 Subject: [PATCH 2/7] CE3: Install custom monitor for law testing suites. Cuts down on noise from the root monitor. --- .../test/scala/io/catbird/util/effect/BaseLawSuite.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala index ac3c1b48..7cb8197e 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala @@ -1,6 +1,7 @@ package io.catbird.util package effect +import com.twitter.util.Monitor import org.scalatest.funsuite.AnyFunSuite import org.scalatest.prop.Configuration import org.typelevel.discipline.scalatest.FunSuiteDiscipline @@ -10,4 +11,9 @@ abstract class BaseLawSuite with FunSuiteDiscipline with Configuration with ArbitraryInstances - with EqInstances + with EqInstances { + protected val monitor = Monitor.mk { case e => println("Monitored: " + e); true } + + override protected def withFixture(test: NoArgTest) = + Monitor.using(monitor)(test()) +} From 2d8664eeeed0171204d5bb41790035bfc1274400 Mon Sep 17 00:00:00 2001 From: Ben Spencer Date: Sun, 23 May 2021 16:31:23 +0100 Subject: [PATCH 3/7] CE3: Remove redundant MonadErrorTests for Future. --- effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala index 1023bd6a..b291c381 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala @@ -3,7 +3,6 @@ package io.catbird.util.effect import cats.Eq import cats.effect.laws.MonadCancelTests import cats.instances.all._ -import cats.laws.discipline.MonadErrorTests import cats.laws.discipline.arbitrary._ import com.twitter.conversions.DurationOps._ import com.twitter.util.Future @@ -14,6 +13,5 @@ class FutureSuite extends BaseLawSuite { implicit def futureEq[A](implicit A: Eq[A]): Eq[Future[A]] = futureEqWithFailure(1.seconds) - checkAll("Future[Int]", MonadErrorTests[Future, Throwable].monadError[Int, Int, Int]) checkAll("Future[Int]", MonadCancelTests[Future, Throwable].monadCancel[Int, Int, Int]) } From 54f443f07e9883ffd449ae7c9aae4f4bde846d1f Mon Sep 17 00:00:00 2001 From: Ben Spencer Date: Sun, 23 May 2021 16:58:15 +0100 Subject: [PATCH 4/7] CE3: Util for testing an individual law with a specific seed. --- .../io/catbird/util/effect/BaseLawSuite.scala | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala index 7cb8197e..b5198e05 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala @@ -3,7 +3,11 @@ package effect import com.twitter.util.Monitor import org.scalatest.funsuite.AnyFunSuite +import org.scalacheck.{ Arbitrary, Prop, Test } +import org.scalacheck.rng.Seed +import org.scalacheck.util.Pretty import org.scalatest.prop.Configuration +import org.scalatestplus.scalacheck.Checkers import org.typelevel.discipline.scalatest.FunSuiteDiscipline abstract class BaseLawSuite @@ -16,4 +20,25 @@ abstract class BaseLawSuite override protected def withFixture(test: NoArgTest) = Monitor.using(monitor)(test()) + + // For debugging an individual failing law. Call this and use `testOnly -- -z withSeed`. Eg: + // + // cats law: + // import cats.kernel.laws.discipline._ + // testLawWithSeed( + // cats.laws.ApplicativeLaws[Rerunnable].applicativeIdentity[Int], + // "NRzb_Wsi6ki82wDgbifBkUvntAPN5kaO8FbKSYpKXiF=", + // ) + // + // cats-effect law: + // testLawWithSeed( + // cats.effect.laws.MonadCancelLaws[Rerunnable, Throwable].uncancelablePollInverseNestIsUncancelable[Int], + // "957izWLFn5kHIug2eCpFHBfajl4zwQGrNdVvC5XWtFK=", + // ) + // + // You may need to wrangle the law into a Function1 with .tupled. + protected def testLawWithSeed[A: Arbitrary: * => Pretty, L: * => Prop](law: A => L, seed: String) = + test("withSeed") { + Checkers.check(Prop.forAll(law), Test.Parameters.default.withInitialSeed(Seed.fromBase64(seed).get)) + } } From 4b0fc29ce91b1d504a24084f0ff3baa9168170bd Mon Sep 17 00:00:00 2001 From: Ben Spencer Date: Sun, 23 May 2021 15:15:11 +0100 Subject: [PATCH 5/7] CE3: Use testkit generators for Rerunnable law tests. --- .../io/catbird/util/effect/BaseLawSuite.scala | 7 +------ .../io/catbird/util/effect/FutureSuite.scala | 4 ++-- .../scala/io/catbird/util/effect/Runners.scala | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala index b5198e05..c9060baf 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/BaseLawSuite.scala @@ -10,12 +10,7 @@ import org.scalatest.prop.Configuration import org.scalatestplus.scalacheck.Checkers import org.typelevel.discipline.scalatest.FunSuiteDiscipline -abstract class BaseLawSuite - extends AnyFunSuite - with FunSuiteDiscipline - with Configuration - with ArbitraryInstances - with EqInstances { +abstract class BaseLawSuite extends AnyFunSuite with FunSuiteDiscipline with Configuration with EqInstances { protected val monitor = Monitor.mk { case e => println("Monitored: " + e); true } override protected def withFixture(test: NoArgTest) = diff --git a/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala index b291c381..d09e8e63 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/FutureSuite.scala @@ -6,9 +6,9 @@ import cats.instances.all._ import cats.laws.discipline.arbitrary._ import com.twitter.conversions.DurationOps._ import com.twitter.util.Future -import io.catbird.util.futureEqWithFailure +import io.catbird.util.{ ArbitraryInstances, futureEqWithFailure } -class FutureSuite extends BaseLawSuite { +class FutureSuite extends BaseLawSuite with ArbitraryInstances { implicit def futureEq[A](implicit A: Eq[A]): Eq[Future[A]] = futureEqWithFailure(1.seconds) diff --git a/effect3/src/test/scala/io/catbird/util/effect/Runners.scala b/effect3/src/test/scala/io/catbird/util/effect/Runners.scala index 4061c304..0a047cef 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/Runners.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/Runners.scala @@ -2,10 +2,11 @@ package io.catbird.util.effect import cats.Eq import cats.effect.{ IO, Outcome, unsafe } +import cats.effect.kernel.testkit.SyncGenerators import cats.effect.testkit.TestContext import cats.effect.unsafe.IORuntimeConfig import io.catbird.util.{ EqInstances, Rerunnable } -import org.scalacheck.Prop +import org.scalacheck.{ Arbitrary, Cogen, Prop } import scala.annotation.implicitNotFound import scala.concurrent.duration.FiniteDuration @@ -17,6 +18,18 @@ import scala.language.implicitConversions */ trait Runners { self: EqInstances => + implicit def arbitraryRerunnable[A: Arbitrary: Cogen]: Arbitrary[Rerunnable[A]] = { + val generators = new SyncGenerators[Rerunnable] { + val F = rerunnableInstance + + val arbitraryE = Arbitrary.arbThrowable + val cogenE = Cogen.cogenThrowable + + val arbitraryFD = Arbitrary.arbFiniteDuration + } + Arbitrary(generators.generators[A]) + } + implicit val ticker: Ticker = Ticker(TestContext()) implicit def eqIOA[A: Eq](implicit ticker: Ticker): Eq[IO[A]] = From 0c22384ba0a4fa2c00f9e10f51a651c0317e745f Mon Sep 17 00:00:00 2001 From: Ben Spencer Date: Tue, 1 Jun 2021 14:24:04 +0100 Subject: [PATCH 6/7] CE3: Catbird-native Runners for tests. This works with Await for Sync: it will need to become more sophisticated to support cancellation and non-termination. --- build.sbt | 2 +- .../io/catbird/util/effect/Runners.scala | 66 +++++-------------- 2 files changed, 16 insertions(+), 52 deletions(-) diff --git a/build.sbt b/build.sbt index 1e447f8c..d80b9754 100644 --- a/build.sbt +++ b/build.sbt @@ -116,7 +116,7 @@ lazy val effect3 = project libraryDependencies ++= Seq( "org.typelevel" %% "cats-effect" % catsEffect3Version, "org.typelevel" %% "cats-effect-laws" % catsEffect3Version % Test, - "org.typelevel" %% "cats-effect-testkit" % catsEffect3Version % Test + "org.typelevel" %% "cats-effect-kernel-testkit" % catsEffect3Version % Test ), (Test / scalacOptions) ~= { _.filterNot(Set("-Yno-imports", "-Yno-predef")) diff --git a/effect3/src/test/scala/io/catbird/util/effect/Runners.scala b/effect3/src/test/scala/io/catbird/util/effect/Runners.scala index 0a047cef..4953f237 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/Runners.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/Runners.scala @@ -1,21 +1,14 @@ package io.catbird.util.effect import cats.Eq -import cats.effect.{ IO, Outcome, unsafe } +import cats.effect.kernel.Outcome import cats.effect.kernel.testkit.SyncGenerators -import cats.effect.testkit.TestContext -import cats.effect.unsafe.IORuntimeConfig +import com.twitter.util.{ Await, Return, Throw } import io.catbird.util.{ EqInstances, Rerunnable } import org.scalacheck.{ Arbitrary, Cogen, Prop } -import scala.annotation.implicitNotFound -import scala.concurrent.duration.FiniteDuration -import scala.concurrent.duration._ import scala.language.implicitConversions -/** - * Test helpers mostly taken from the cats-effect IOSpec. - */ trait Runners { self: EqInstances => implicit def arbitraryRerunnable[A: Arbitrary: Cogen]: Arbitrary[Rerunnable[A]] = { @@ -30,48 +23,19 @@ trait Runners { self: EqInstances => Arbitrary(generators.generators[A]) } - implicit val ticker: Ticker = Ticker(TestContext()) - - implicit def eqIOA[A: Eq](implicit ticker: Ticker): Eq[IO[A]] = + implicit def eqRerunnableA[A: Eq]: Eq[Rerunnable[A]] = Eq.by(unsafeRun(_)) - implicit def rerunnableEq[A](implicit A: Eq[A]): Eq[Rerunnable[A]] = - Eq.by[Rerunnable[A], IO[A]](rerunnableToIO) - - implicit def boolRunnings(rerunnableB: Rerunnable[Boolean])(implicit ticker: Ticker): Prop = - Prop(unsafeRun(rerunnableToIO(rerunnableB)).fold(false, _ => false, _.getOrElse(false))) - - def unsafeRun[A](ioa: IO[A])(implicit ticker: Ticker): Outcome[Option, Throwable, A] = - try { - var results: Outcome[Option, Throwable, A] = Outcome.Succeeded(None) - - ioa.unsafeRunAsync { - case Left(t) => results = Outcome.Errored(t) - case Right(a) => results = Outcome.Succeeded(Some(a)) - }(unsafe.IORuntime(ticker.ctx, ticker.ctx, scheduler, () => (), IORuntimeConfig())) - - ticker.ctx.tickAll(1.days) - - results - } catch { - case t: Throwable => - t.printStackTrace() - throw t - } - - def scheduler(implicit ticker: Ticker): unsafe.Scheduler = - new unsafe.Scheduler { - import ticker.ctx - - def sleep(delay: FiniteDuration, action: Runnable): Runnable = { - val cancel = ctx.schedule(delay, action) - new Runnable { def run() = cancel() } - } - - def nowMillis() = ctx.now().toMillis - def monotonicNanos() = ctx.now().toNanos - } - - @implicitNotFound("could not find an instance of Ticker; try using `in ticked { implicit ticker =>`") - case class Ticker(ctx: TestContext) + implicit def boolRunnings(rerunnableB: Rerunnable[Boolean]): Prop = + Prop(unsafeRun(rerunnableB).fold(false, _ => false, _.getOrElse(false))) + + def unsafeRun[A](fa: Rerunnable[A]): Outcome[Option, Throwable, A] = + Await.result( + fa.liftToTry + .map[Outcome[Option, Throwable, A]] { + case Return(a) => Outcome.succeeded(Some(a)) + case Throw(e) => Outcome.errored(e) + } + .run + ) } From 57b83f39b330b9ae09bf2132d8afeb2b06bd8a97 Mon Sep 17 00:00:00 2001 From: Ben Spencer Date: Tue, 1 Jun 2021 13:55:41 +0100 Subject: [PATCH 7/7] CE3: Use twitter-util Time and Stopwatch for Clock implementation. Allows using the twitter-util testing facilities. --- .../util/effect/RerunnableInstances.scala | 7 ++- .../util/effect/RerunnableClockSuite.scala | 48 ------------------- .../catbird/util/effect/RerunnableSuite.scala | 20 +++++++- 3 files changed, 21 insertions(+), 54 deletions(-) delete mode 100644 effect3/src/test/scala/io/catbird/util/effect/RerunnableClockSuite.scala diff --git a/effect3/src/main/scala/io/catbird/util/effect/RerunnableInstances.scala b/effect3/src/main/scala/io/catbird/util/effect/RerunnableInstances.scala index df6ff12e..ed7d9195 100644 --- a/effect3/src/main/scala/io/catbird/util/effect/RerunnableInstances.scala +++ b/effect3/src/main/scala/io/catbird/util/effect/RerunnableInstances.scala @@ -2,12 +2,11 @@ package io.catbird.util.effect import cats.effect.Clock import cats.effect.kernel.{ MonadCancel, Outcome, Sync } -import com.twitter.util.{ Future, Monitor } +import com.twitter.util.{ Future, Monitor, Stopwatch, Time } import io.catbird.util.{ Rerunnable, RerunnableMonadError } import java.lang.Throwable import java.util.concurrent.TimeUnit -import java.lang.System import scala.Unit import scala.concurrent.duration.FiniteDuration @@ -24,10 +23,10 @@ trait RerunnableInstances { Rerunnable(thunk) final override def realTime: Rerunnable[FiniteDuration] = - Rerunnable(FiniteDuration(System.currentTimeMillis(), TimeUnit.MILLISECONDS)) + Rerunnable(FiniteDuration(Time.nowNanoPrecision.inNanoseconds, TimeUnit.NANOSECONDS)) final override def monotonic: Rerunnable[FiniteDuration] = - Rerunnable(FiniteDuration(System.nanoTime(), TimeUnit.NANOSECONDS)) + Rerunnable(FiniteDuration(Stopwatch.timeNanos(), TimeUnit.NANOSECONDS)) final override def forceR[A, B](fa: Rerunnable[A])(fb: Rerunnable[B]): Rerunnable[B] = fa.liftToTry.flatMap { resultA => diff --git a/effect3/src/test/scala/io/catbird/util/effect/RerunnableClockSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/RerunnableClockSuite.scala deleted file mode 100644 index c1011ac0..00000000 --- a/effect3/src/test/scala/io/catbird/util/effect/RerunnableClockSuite.scala +++ /dev/null @@ -1,48 +0,0 @@ -package io.catbird.util.effect - -import java.time.Instant -import java.util.concurrent.TimeUnit - -import cats.effect.Clock -import com.twitter.util.Await -import io.catbird.util.Rerunnable -import org.scalatest.{ Outcome } -import org.scalatest.concurrent.{ Eventually, IntegrationPatience } -import org.scalatest.funsuite.FixtureAnyFunSuite - -/** - * We'll use `eventually` and a reasonably big tolerance here to prevent CI from failing if it is a bit slow. - * - * Technically the implementation is just an extremely thin wrapper around `System.currentTimeMillis()` - * and `System.nanoTime()` so as long as the result is the same order of magnitude (and therefore the - * unit-conversion is correct) we should be fine. - */ -class RerunnableClockSuite extends FixtureAnyFunSuite with Eventually with IntegrationPatience { - - protected final class FixtureParam { - def now: Instant = Instant.now() - } - - test("Retrieval of real time") { f => - eventually { - val result = Await.result( - Clock[Rerunnable].realTime.map(duration => Instant.ofEpochMilli(duration.toMillis)).run - ) - - assert(java.time.Duration.between(result, f.now).abs().toMillis < 50) - } - } - - test("Retrieval of monotonic time") { f => - eventually { - val result = Await.result( - Clock[Rerunnable].monotonic.map(duration => duration.toNanos).run - ) - - val durationBetween = Math.abs(System.nanoTime() - result) - assert(TimeUnit.MILLISECONDS.convert(durationBetween, TimeUnit.NANOSECONDS) < 5) - } - } - - override protected def withFixture(test: OneArgTest): Outcome = withFixture(test.toNoArgTest(new FixtureParam)) -} diff --git a/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala b/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala index fd624f3b..2e777685 100644 --- a/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala +++ b/effect3/src/test/scala/io/catbird/util/effect/RerunnableSuite.scala @@ -1,6 +1,6 @@ package io.catbird.util.effect -import cats.effect.MonadCancel +import cats.effect.kernel.{ Clock, MonadCancel, Outcome } import cats.effect.kernel.testkit.SyncTypeGenerators import cats.effect.laws.SyncTests import cats.instances.either._ @@ -8,7 +8,7 @@ import cats.instances.int._ import cats.instances.tuple._ import cats.instances.unit._ import cats.laws.discipline.arbitrary._ -import com.twitter.util.{ Await, Monitor, Throw } +import com.twitter.util.{ Await, Monitor, Throw, Time } import io.catbird.util.Rerunnable class RerunnableSuite extends BaseLawSuite with SyncTypeGenerators with Runners { @@ -16,6 +16,22 @@ class RerunnableSuite extends BaseLawSuite with SyncTypeGenerators with Runners // This includes tests for Clock, MonadCancel, and MonadError checkAll("Rerunnable[Int]", SyncTests[Rerunnable].sync[Int, Int, Int]) + test("Retrieval of real time") { + val nanos = 123456789L + val result = Time.withTimeAt(Time.fromNanoseconds(nanos)) { _ => + unsafeRun(Clock[Rerunnable].realTime.map(_.toNanos)) + } + assert(result == Outcome.succeeded(Some(nanos))) + } + + test("Retrieval of monotonic time") { + val nanos = 123456789L + val result = Time.withTimeAt(Time.fromNanoseconds(nanos)) { _ => + unsafeRun(Clock[Rerunnable].monotonic.map(_.toNanos)) + } + assert(result == Outcome.succeeded(Some(nanos))) + } + test("Exceptions thrown by release are handled by Monitor") { val useException = new Exception("thrown by use") val releaseException = new Exception("thrown by release")