From c273eee4f077f2e83547164591b98b64402c03cb Mon Sep 17 00:00:00 2001 From: Heather Miller Date: Wed, 13 Aug 2014 14:14:34 -0700 Subject: [PATCH 1/3] Adds ability to pickle/unpickle things marked @transient --- core/src/main/scala/pickling/Macros.scala | 22 ++++++++++- core/src/main/scala/pickling/ir/IRs.scala | 38 +++++++++++++++++-- .../test/scala/pickling/run/transient.scala | 19 ++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 core/src/test/scala/pickling/run/transient.scala diff --git a/core/src/main/scala/pickling/Macros.scala b/core/src/main/scala/pickling/Macros.scala index 912539578b..35f53a93dd 100644 --- a/core/src/main/scala/pickling/Macros.scala +++ b/core/src/main/scala/pickling/Macros.scala @@ -258,8 +258,26 @@ trait UnpicklerMacros extends Macro { val someConstructorIsPrivate = ctors.exists(_.isPrivate) // println(s"someConstructorIsPrivate: $someConstructorIsPrivate") - val canCallCtor = !someConstructorIsPrivate && !cir.fields.exists(_.isErasedParam) && isPreciseType - // println(s"canCallCtor: $canCallCtor") + val canCallCtor = !someConstructorIsPrivate && !cir.fields.exists(_.isErasedParam) && isPreciseType && { + // there must not be a transient ctor param + // STEP 1: we need to figure out if there is a transient ctor param + val primaryCtor = tpe.declaration(nme.CONSTRUCTOR) match { + case overloaded: TermSymbol => overloaded.alternatives.head.asMethod // NOTE: primary ctor is always the first in the list + case primaryCtor: MethodSymbol => primaryCtor + case NoSymbol => NoSymbol + } + + val allAccessors = tpe.declarations.collect { case meth: MethodSymbol if meth.isAccessor || meth.isParamAccessor => meth } + + val (filteredAccessors, transientAccessors) = allAccessors.partition(irs.notMarkedTransient) // shoulsai iaobjsobj aaaobjsecta insteeyd m d daybe + + val hasTransientParam = (primaryCtor != NoSymbol) && primaryCtor.asMethod.paramss.flatten.exists { sym => + transientAccessors.exists(acc => acc.name.toString == sym.name.toString) + } + + !hasTransientParam + } + // STEP 2: remove transient fields from the "pending fields", the fields that need to be restored. // TODO: for ultimate loop safety, pendingFields should be hoisted to the outermost unpickling scope // For example, in the snippet below, when unpickling C, we'll need to move the b.c assignment not diff --git a/core/src/main/scala/pickling/ir/IRs.scala b/core/src/main/scala/pickling/ir/IRs.scala index ff6c190e92..cc74c1b576 100644 --- a/core/src/main/scala/pickling/ir/IRs.scala +++ b/core/src/main/scala/pickling/ir/IRs.scala @@ -33,14 +33,43 @@ class IRs[U <: Universe with Singleton](val uni: U) { private type C = ClassIR // TODO: minimal versus verbose PickleFormat. i.e. someone might want all concrete inherited fields in their pickle + + def notMarkedTransient(sym: TermSymbol): Boolean = { + //println(s"checking annots of ${sym.toString}...") + val tr = scala.util.Try { + if (sym.accessed != NoSymbol) { + val overall = sym.accessed.annotations.forall { a => + val res = (a.tpe =:= typeOf[scala.transient]) + !res + } + overall + } else true // if there is no backing field, then it cannot be marked transient + } + if (tr.isFailure) { + } + tr.isFailure || tr.get + } + + /** Creates FieldIRs for the given type, tp. + */ private def fields(tp: Type): Q = { val ctor = tp.declaration(nme.CONSTRUCTOR) match { case overloaded: TermSymbol => overloaded.alternatives.head.asMethod // NOTE: primary ctor is always the first in the list case primaryCtor: MethodSymbol => primaryCtor case NoSymbol => NoSymbol } - val ctorParams = if (ctor != NoSymbol) ctor.asMethod.paramss.flatten.map(_.asTerm) else Nil - val allAccessors = tp.declarations.collect{ case meth: MethodSymbol if meth.isAccessor || meth.isParamAccessor => meth } + + val allAccessors = tp.declarations.collect { case meth: MethodSymbol if meth.isAccessor || meth.isParamAccessor => meth } + + val (filteredAccessors, transientAccessors) = allAccessors.partition(notMarkedTransient) + + val ctorParams = if (ctor != NoSymbol) ctor.asMethod.paramss.flatten.flatMap { sym => + if (transientAccessors.exists(acc => acc.name.toString == sym.name.toString)) { + //println(s"found a BAD accessor: $sym") + List() + } else List(sym.asTerm) + } else Nil + val (paramAccessors, otherAccessors) = allAccessors.partition(_.isParamAccessor) def mkFieldIR(sym: TermSymbol, param: Option[TermSymbol], accessor: Option[MethodSymbol]) = { @@ -54,7 +83,8 @@ class IRs[U <: Universe with Singleton](val uni: U) { val varGetters = otherAccessors.collect{ case meth if meth.isGetter && meth.accessed != NoSymbol && meth.accessed.asTerm.isVar => meth } val varFields = varGetters.map(sym => mkFieldIR(sym, None, Some(sym))) - paramFields ++ varFields + val res = paramFields ++ varFields + res } private def composition(f1: (Q, Q) => Q, f2: (C, C) => C, f3: C => List[C]) = @@ -62,7 +92,7 @@ class IRs[U <: Universe with Singleton](val uni: U) { private val f1 = (q1: Q, q2: Q) => q1 ++ q2 - private val f2 = (c1: C, c2: C) => ClassIR(c2.tpe, c1, fields(c2.tpe)) + private val f2 = (c1: C, c2: C) => ClassIR(c2.tpe, c1, /*fields(c2.tpe)*/c2.fields) // here: fields is called a 2nd time. private val f3 = (c: C) => c.tpe.baseClasses diff --git a/core/src/test/scala/pickling/run/transient.scala b/core/src/test/scala/pickling/run/transient.scala new file mode 100644 index 0000000000..3c765a701f --- /dev/null +++ b/core/src/test/scala/pickling/run/transient.scala @@ -0,0 +1,19 @@ +package scala.pickling.transienttest + +import org.scalatest.FunSuite +import scala.pickling._ +import json._ + +case class Person(val name: String , @transient val ssNumber: Int) { + override def toString = s"Person($name)" +} + +class TransientSimpleTest extends FunSuite { + test("main") { + val per = Person("Jenny", 123) + val p: JSONPickle = per.pickle + val up = p.unpickle[Person] + assert(up.ssNumber == 0) + assert(per.toString == up.toString) + } +} \ No newline at end of file From 8763c7ba21eb3a14f43683888b4632f94275bbe9 Mon Sep 17 00:00:00 2001 From: Philipp Haller Date: Wed, 13 Aug 2014 23:35:26 +0200 Subject: [PATCH 2/3] Clean-ups in IRs --- core/src/main/scala/pickling/ir/IRs.scala | 24 ++++++----------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/core/src/main/scala/pickling/ir/IRs.scala b/core/src/main/scala/pickling/ir/IRs.scala index cc74c1b576..377a786862 100644 --- a/core/src/main/scala/pickling/ir/IRs.scala +++ b/core/src/main/scala/pickling/ir/IRs.scala @@ -24,7 +24,6 @@ class IRs[U <: Universe with Singleton](val uni: U) { // this part is interesting to unpicklers def hasSetter = setter.isDefined def isErasedParam = isParam && accessor.isEmpty // TODO: this should somehow communicate with the constructors phase! - def isReifiedParam = isParam && accessor.nonEmpty def isNonParam = !isParam } case class ClassIR(tpe: Type, parent: ClassIR, fields: List[FieldIR]) extends PickleIR @@ -35,17 +34,9 @@ class IRs[U <: Universe with Singleton](val uni: U) { // TODO: minimal versus verbose PickleFormat. i.e. someone might want all concrete inherited fields in their pickle def notMarkedTransient(sym: TermSymbol): Boolean = { - //println(s"checking annots of ${sym.toString}...") val tr = scala.util.Try { - if (sym.accessed != NoSymbol) { - val overall = sym.accessed.annotations.forall { a => - val res = (a.tpe =:= typeOf[scala.transient]) - !res - } - overall - } else true // if there is no backing field, then it cannot be marked transient - } - if (tr.isFailure) { + (sym.accessed == NoSymbol) || // if there is no backing field, then it cannot be marked transient + !sym.accessed.annotations.exists(_.tpe =:= typeOf[scala.transient]) } tr.isFailure || tr.get } @@ -64,10 +55,8 @@ class IRs[U <: Universe with Singleton](val uni: U) { val (filteredAccessors, transientAccessors) = allAccessors.partition(notMarkedTransient) val ctorParams = if (ctor != NoSymbol) ctor.asMethod.paramss.flatten.flatMap { sym => - if (transientAccessors.exists(acc => acc.name.toString == sym.name.toString)) { - //println(s"found a BAD accessor: $sym") - List() - } else List(sym.asTerm) + if (transientAccessors.exists(acc => acc.name.toString == sym.name.toString)) List() + else List(sym.asTerm) } else Nil val (paramAccessors, otherAccessors) = allAccessors.partition(_.isParamAccessor) @@ -83,8 +72,7 @@ class IRs[U <: Universe with Singleton](val uni: U) { val varGetters = otherAccessors.collect{ case meth if meth.isGetter && meth.accessed != NoSymbol && meth.accessed.asTerm.isVar => meth } val varFields = varGetters.map(sym => mkFieldIR(sym, None, Some(sym))) - val res = paramFields ++ varFields - res + paramFields ++ varFields } private def composition(f1: (Q, Q) => Q, f2: (C, C) => C, f3: C => List[C]) = @@ -92,7 +80,7 @@ class IRs[U <: Universe with Singleton](val uni: U) { private val f1 = (q1: Q, q2: Q) => q1 ++ q2 - private val f2 = (c1: C, c2: C) => ClassIR(c2.tpe, c1, /*fields(c2.tpe)*/c2.fields) // here: fields is called a 2nd time. + private val f2 = (c1: C, c2: C) => ClassIR(c2.tpe, c1, c2.fields) private val f3 = (c: C) => c.tpe.baseClasses From c703d5d53d3d59adf5044d96914391eac9df6825 Mon Sep 17 00:00:00 2001 From: Heather Miller Date: Wed, 13 Aug 2014 16:18:07 -0700 Subject: [PATCH 3/3] Handles transient now. Handles pickling ClassTags too. + tests --- core/src/main/scala/pickling/Compat.scala | 6 ++++ core/src/main/scala/pickling/FastTags.scala | 14 ++++++++ core/src/main/scala/pickling/ir/IRs.scala | 12 ++----- .../scala/pickling/run/implicit-params.scala | 21 ++++++++++++ .../test/scala/pickling/run/transient.scala | 32 ++++++++++++++++++- 5 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 core/src/test/scala/pickling/run/implicit-params.scala diff --git a/core/src/main/scala/pickling/Compat.scala b/core/src/main/scala/pickling/Compat.scala index 583aea7b1d..30a0486033 100644 --- a/core/src/main/scala/pickling/Compat.scala +++ b/core/src/main/scala/pickling/Compat.scala @@ -85,6 +85,12 @@ object Compat { c.Expr[FastTypeTag[T]](bundle.impl[T]) } + def FastTypeTagMacros_implClassTag[T: c.WeakTypeTag](c: Context): c.Expr[FastTypeTag[T]] = { + val c0: c.type = c + val bundle = new { val c: c0.type = c0 } with FastTypeTagMacros + c.Expr[FastTypeTag[T]](bundle.implClassTag[T]) + } + def FastTypeTagMacros_apply(c: Context)(key: c.Expr[String]): c.Expr[FastTypeTag[t]] forSome { type t } = { val c0: c.type = c val bundle = new { val c: c0.type = c0 } with FastTypeTagMacros diff --git a/core/src/main/scala/pickling/FastTags.scala b/core/src/main/scala/pickling/FastTags.scala index fd1e91ae75..31e4bd6c49 100644 --- a/core/src/main/scala/pickling/FastTags.scala +++ b/core/src/main/scala/pickling/FastTags.scala @@ -5,6 +5,7 @@ import scala.reflect.macros.Context import scala.reflect.api.{Universe => ApiUniverse} import scala.reflect.runtime.{universe => ru} import language.experimental.macros +import scala.reflect.ClassTag trait FastTypeTag[T] extends Equals { def mirror: ru.Mirror @@ -19,6 +20,8 @@ trait FastTypeTag[T] extends Equals { object FastTypeTag { implicit def materializeFastTypeTag[T]: FastTypeTag[T] = macro Compat.FastTypeTagMacros_impl[T] + implicit def materializeFastTypeTagOfClassTag[T]: FastTypeTag[ClassTag[T]] = macro Compat.FastTypeTagMacros_implClassTag[T] + private def stdTag[T: ru.TypeTag]: FastTypeTag[T] = apply(scala.reflect.runtime.currentMirror, ru.typeOf[T], ru.typeOf[T].key).asInstanceOf[FastTypeTag[T]] implicit val Null = stdTag[Null] implicit val Byte = stdTag[Byte] @@ -108,6 +111,17 @@ trait FastTypeTagMacros extends Macro { } """ } + def implClassTag[T: c.WeakTypeTag]: c.Tree = { + import c.universe._ + val T = weakTypeOf[T] + q""" + new FastTypeTag[ClassTag[$T]] { + def mirror = scala.pickling.internal.`package`.currentMirror + lazy val tpe = scala.reflect.runtime.universe.weakTypeOf[ClassTag[$T]] + def key = "ClassTag[" + ${T.key} + "]" + } + """ + } def apply(key: c.Tree): c.Tree = { import c.universe._ q"""scala.pickling.FastTypeTag(scala.pickling.internal.`package`.currentMirror, $key)""" diff --git a/core/src/main/scala/pickling/ir/IRs.scala b/core/src/main/scala/pickling/ir/IRs.scala index cc74c1b576..147e10f758 100644 --- a/core/src/main/scala/pickling/ir/IRs.scala +++ b/core/src/main/scala/pickling/ir/IRs.scala @@ -35,17 +35,9 @@ class IRs[U <: Universe with Singleton](val uni: U) { // TODO: minimal versus verbose PickleFormat. i.e. someone might want all concrete inherited fields in their pickle def notMarkedTransient(sym: TermSymbol): Boolean = { - //println(s"checking annots of ${sym.toString}...") val tr = scala.util.Try { - if (sym.accessed != NoSymbol) { - val overall = sym.accessed.annotations.forall { a => - val res = (a.tpe =:= typeOf[scala.transient]) - !res - } - overall - } else true // if there is no backing field, then it cannot be marked transient - } - if (tr.isFailure) { + (sym.accessed == NoSymbol) || // if there is no backing field, then it cannot be marked transient + !sym.accessed.annotations.exists(_.tpe =:= typeOf[scala.transient]) } tr.isFailure || tr.get } diff --git a/core/src/test/scala/pickling/run/implicit-params.scala b/core/src/test/scala/pickling/run/implicit-params.scala new file mode 100644 index 0000000000..cd1f397795 --- /dev/null +++ b/core/src/test/scala/pickling/run/implicit-params.scala @@ -0,0 +1,21 @@ +package scala.pickling.implicitparams + +import org.scalatest.FunSuite +import scala.pickling._ +import json._ + + +case class Person(implicit name: String, age: Int) +object Test { + class SimpleImplParamTest extends FunSuite { + test("main") { + implicit val nme = "Harry" + implicit val age = 18 + + val per = Person() + val p = per.pickle + val up = p.unpickle[Person] + assert(per == up) + } + } +} diff --git a/core/src/test/scala/pickling/run/transient.scala b/core/src/test/scala/pickling/run/transient.scala index 3c765a701f..7fe78c0f07 100644 --- a/core/src/test/scala/pickling/run/transient.scala +++ b/core/src/test/scala/pickling/run/transient.scala @@ -1,6 +1,7 @@ package scala.pickling.transienttest import org.scalatest.FunSuite +import scala.reflect.ClassTag import scala.pickling._ import json._ @@ -8,6 +9,23 @@ case class Person(val name: String , @transient val ssNumber: Int) { override def toString = s"Person($name)" } +class Dependency[T] + +class SparkConf(loadDefaults: Boolean) +class SparkContext(config: SparkConf) + +class RDD[T: ClassTag]( + @transient private var sc: SparkContext, + @transient private var deps: Seq[Dependency[_]] + ) + +class RangePartitioner[K : ClassTag, V]( + @transient val partitions: Int, + @transient val rdd: RDD[_ <: Product2[K,V]], + private var ascending: Boolean = true) { + override def toString = s"RangePartitioner(ascending = $ascending)" +} + class TransientSimpleTest extends FunSuite { test("main") { val per = Person("Jenny", 123) @@ -16,4 +34,16 @@ class TransientSimpleTest extends FunSuite { assert(up.ssNumber == 0) assert(per.toString == up.toString) } -} \ No newline at end of file +} + +class TransientSparkTest extends FunSuite { + test("main") { + val sc = new SparkContext(new SparkConf(true)) + val rdd = new RDD[(Int, Int)](sc, Seq(new Dependency())) + val rp = new RangePartitioner[Int, Int](2, rdd) + val p: JSONPickle = rp.pickle + val up = p.unpickle[RangePartitioner[Int, Int]] + assert(rp.toString == up.toString) + } +} +