Support synthesis of JavaScript programs (fuzzing) (#269)

.completion

@@ -11,7 +11,7 @@ _esmeta_completions() {
   local cur prev opts lastc informats outformats datafiles
   cur="${COMP_WORDS[COMP_CWORD]}"
   prev="${COMP_WORDS[COMP_CWORD-1]}"
-  cmdList="help extract compile build-cfg tycheck parse eval web test262-test inject mutate"
+  cmdList="help extract compile build-cfg tycheck parse eval web test262-test fuzz inject mutate"
   globalOpt="-silent -error -status -time -test262dir"
   helpOpt=""
   extractOpt="-extract:target -extract:log -extract:eval -extract:repl"
@@ -22,6 +22,7 @@ _esmeta_completions() {
   evalOpt="-eval:timeout -eval:multiple -eval:log -eval:detail-log"
   webOpt="-web:port"
   test262testOpt="-test262-test:target -test262-test:features -test262-test:progress -test262-test:coverage -test262-test:timeout -test262-test:with-yet -test262-test:log -test262-test:detail-log -test262-test:concurrent"
+  fuzzOpt="-fuzz:log -fuzz:log-interval -fuzz:out -fuzz:debug -fuzz:timeout -fuzz:trial -fuzz:duration -fuzz:seed -fuzz:cp -fuzz:init -fuzz:k-fs"
   injectOpt="-inject:defs -inject:out -inject:log"
   mutateOpt="-mutate:out -mutate:mutator -mutate:untilValid"
   # completion for commands
@@ -72,6 +73,10 @@ _esmeta_completions() {
           COMPREPLY=($(compgen -W "${globalOpt} ${extractOpt} ${compileOpt} ${buildcfgOpt} ${test262testOpt}"))
           return 0
           ;;
+        fuzz)
+          COMPREPLY=($(compgen -W "${globalOpt} ${extractOpt} ${compileOpt} ${buildcfgOpt} ${fuzzOpt}"))
+          return 0
+          ;;
         inject)
           COMPREPLY=($(compgen -W "${globalOpt} ${extractOpt} ${compileOpt} ${buildcfgOpt} ${injectOpt}"))
           return 0

README.md

@@ -25,7 +25,7 @@ automatically generates language-based tools.
     + [Interactive Execution with ECMAScript Double Debugger](#interactive-execution-with-ecmascript-double-debugger)
     + [Conformance Test Synthesizer from ECMA-262](#conformance-test-synthesizer-from-ecma-262)
     + [Type Analysis on ECMA-262](#type-analysis-on-ecma-262)
-    + [Meta-Level Static Analyzer for ECMAScript](#meta-level-static-analyzer-for-ecmascript) (temporarily removed)
+    + ~~[Meta-Level Static Analyzer for ECMAScript](#meta-level-static-analyzer-for-ecmascript)~~ (temporarily removed)
   * [Academic Achievement](#academic-achievement)
     + [Publications](#publications)
     + [PLDI 2022 Tutorial](#pldi-2022-tutorial)
@@ -35,9 +35,9 @@ automatically generates language-based tools.
 ## Installation Guide
 
 We explain how to install ESMeta with the necessary environment settings from
-scratch. Our framework is developed in Scala, which works on JDK 8+, including
-GraalVM. So before installation, please install [JDK
-8+](https://www.oracle.com/java/technologies/downloads/) and
+scratch. Our framework is developed in Scala, which works on JDK 17+. So before
+installation, please install [JDK
+17+](https://www.oracle.com/java/technologies/downloads/) and
 [sbt](https://www.scala-sbt.org/), an interactive build tool for Scala.
 
 
@@ -245,16 +245,54 @@ We will enhance it with the following features:
 ### Conformance Test Synthesizer from ECMA-262
 
 ESMeta supports the synthesis of JavaScript files as conformance tests. We
-introduced the main concept of the test synthesis in the [ICSE
-2021 paper](https://doi.org/10.1109/ICSE43902.2021.00015) with a tool named
+introduced the main concept of the test synthesis in the [ICSE 2021
+paper](https://doi.org/10.1109/ICSE43902.2021.00015) with a tool named
 [JEST](https://github.com/kaist-plrg/jest), a **J**avaScript **E**ngines and
 **S**pecification **T**ester. The test synthesis technique consists of two
-parts: 1) _program synthesis_ of JavaScript files and 2) _assertion injection_
-based on the mechanized specification extract from ECMA-262.
+parts: 1) _program synthesis_ of JavaScript programs using **specification
+coverage** and 2) _assertion injection_ based on the mechanized specification
+extract from ECMA-262.
 
-The current version of ESMeta focuses on the assertion injection to a given
-JavaScript file. If you want to inject assertions into the program conforming to
-ECMA-262, please use the `inject` command:
+#### Synthesis of JavaScript Programs
+
+If you want to synthesize JavaScript programs, please use the `fuzz` command:
+```bash
+esmeta fuzz -fuzz:log
+```
+It basically uses the **node/branch coverage** in the mechanized specification
+to synthesize JavaScript programs. The `-fuzz:log` option dumps the synthesized
+JavaScript programs into the `logs/fuzz/fuzz-<date>` directory with the detailed
+information of the synthesis process:
+
+* `seed`: seed of the random number generator
+* `node-coverage.json`: node coverage information
+* `branch-coverage.json`: branch coverage information
+* `constructor.json`: constructor information
+* `mutation-stat.tsv`: statistics of mutation methods
+* `selector-stat.tsv`: statistics of selector methods
+* `summary.tsv`: summary of the synthesis process for each logging interval
+* `target-conds.json`: target conditions for the synthesis
+* `unreach-funcs`: unreachable functions
+* `version`: ESMeta version information
+
+In addition, you can use **feature-sensitive coverage**, which is introduced in
+the [PLDI 2023 paper](https://doi.org/10.1145/3591240), with the following
+options:
+
+* `-fuzz:k-fs=<int>`: the depth of features for feature-sensitive coverage
+* `-fuzz:cp`: use the call path for feature-sensitive coverage
+
+For example, you can synthesize JavaScript programs with the 2-FCPS coverage
+(2-feature-sensitive coverage with call path) as follows:
+```bash
+esmeta fuzz -fuzz:log -fuzz:k-fs=2 -fuzz:cp
+```
+
+#### Assertion Injection
+
+You can inject assertions into the synthesized JavaScript programs according to
+the mechanized specification. If you want to inject assertions, please use the
+`inject` command:
 ```bash
 # inject assertions based on the semantics described in ECMA-262
 $ esmeta inject example.js
@@ -283,9 +321,6 @@ $ esmeta inject example.js -silent -inject:defs -inject:out=test.js
 # - Dumped an assertion-injected ECMAScript program into test.js.
 ```
 
-In the future version of ESMeta, we plan to support the program synthesis
-feature as well.
-
 
 ### Type Analysis on ECMA-262
 
@@ -323,9 +358,9 @@ $ esmeta tycheck -extract:target=2c78e6f
 ```
 
 
-### Meta-Level Static Analyzer for ECMAScript
+### ~~Meta-Level Static Analyzer for ECMAScript~~
 
-> [!NOTE]
+> [!WARNING]
 >
 > The meta-level static analyzer is temporarily removed from the current version
 > of ESMeta. We are working on the improvement of the meta-level static analyzer

build.sbt

@@ -118,6 +118,11 @@ lazy val esParseTest =
 lazy val esAnalyzeTest =
   taskKey[Unit]("Launch analyze tests for ECMAScript (small)")
 
+// injector
+lazy val injectorTest = taskKey[Unit]("Launch injector tests")
+lazy val injectorStringifyTest =
+  taskKey[Unit]("Launch stringify tests for injector (tiny)")
+
 // test262
 lazy val test262ParseTest =
   taskKey[Unit]("Launch parse tests for Test262 (large)")
@@ -158,8 +163,11 @@ lazy val root = project
       "org.apache.commons" % "commons-text" % "1.9",
       "org.jsoup" % "jsoup" % "1.14.3",
       "org.jline" % "jline" % "3.13.3",
+      "org.graalvm.polyglot" % "js" % "24.1.1" pomOnly (),
       ("org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2")
         .cross(CrossVersion.for3Use2_13),
+      ("org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4")
+        .cross(CrossVersion.for3Use2_13),
       ("com.typesafe.akka" %% "akka-actor-typed" % AkkaVersion)
         .cross(CrossVersion.for3Use2_13),
       ("com.typesafe.akka" %% "akka-stream" % AkkaVersion)
@@ -187,6 +195,16 @@ lazy val root = project
     assembly / test := {},
     assembly / assemblyOutputPath := file("bin/esmeta"),
 
+    // fix deduplicate issue of polyglot dependencies
+    // https://stackoverflow.com/questions/54834125/sbt-assembly-deduplicate-module-info-class
+    assembly / assemblyMergeStrategy := {
+      case PathList("module-info.class")               => MergeStrategy.last
+      case path if path.endsWith("/module-info.class") => MergeStrategy.last
+      case x =>
+        val oldStrategy = (assembly / assemblyMergeStrategy).value
+        oldStrategy(x)
+    },
+
     /** tasks for tests */
     // basic tests
     basicTest := (Test / testOnly)
@@ -265,6 +283,11 @@ lazy val root = project
     esEvalTest := (Test / testOnly).toTask(" *.es.Eval*Test").value,
     esParseTest := (Test / testOnly).toTask(" *.es.Parse*Test").value,
     esAnalyzeTest := (Test / testOnly).toTask(" *.es.Analyze*Test").value,
+    // ir
+    injectorTest := (Test / testOnly).toTask(" *.injector.*Test").value,
+    injectorStringifyTest := (Test / testOnly)
+      .toTask(" *.injector.Stringify*Test")
+      .value,
     // test262
     test262ParseTest := (Test / testOnly).toTask(" *.test262.Parse*Test").value,
     test262EvalTest := (Test / testOnly).toTask(" *.test262.Eval*Test").value,

src/main/resources/assertions.js

@@ -1,5 +1,3 @@
-"use strict";
-
 // hidden constructors
 var AsyncArrowFunction = Object.getPrototypeOf(async () => {}).constructor;
 var AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;

src/main/scala/esmeta/Command.scala

@@ -161,8 +161,17 @@ case object CmdTest262Test
 }
 
 // -----------------------------------------------------------------------------
-// ECMAScript Transformer
+// ECMAScript Fuzzer
 // -----------------------------------------------------------------------------
+/** `fuzz` command */
+case object CmdFuzz extends Command("fuzz", CmdBuildCFG >> Fuzz) {
+  val help = "generate ECMAScript programs for fuzzing."
+  val examples = List(
+    "esmeta fuzz                 # generate ECMAScript programs for fuzzing",
+    "esmeta fuzz -fuzz:log       # fuzz in the logging mode.",
+  )
+}
+
 /** `inject` command */
 case object CmdInject extends Command("inject", CmdBuildCFG >> Inject) {
   val help = "injects assertions to check final state of an ECMAScript file."

src/main/scala/esmeta/ESMeta.scala

@@ -75,7 +75,8 @@ object ESMeta extends Git(BASE_DIR) {
     CmdWeb,
     // Tester for Test262 (ECMAScript Test Suite)
     CmdTest262Test,
-    // ECMAScript Transformer
+    // ECMAScript Fuzzer
+    CmdFuzz,
     CmdInject,
     CmdMutate,
   )
@@ -98,7 +99,8 @@ object ESMeta extends Git(BASE_DIR) {
     Web,
     // Tester for Test262 (ECMAScript Test Suite)
     Test262Test,
-    // ECMAScript Transformer
+    // ECMAScript Fuzzer
+    Fuzz,
     Inject,
     Mutate,
   )

src/main/scala/esmeta/analyzer/tychecker/AbsState.scala

@@ -187,7 +187,7 @@ trait AbsStateDecl { self: TyChecker =>
       case One(k) =>
         (for {
           prod <- cfg.grammar.nameMap.get(name)
-          rhs <- prod.rhsList.lift(idx)
+          rhs <- prod.rhsVec.lift(idx)
           nt <- rhs.nts.lift(k.toInt)
         } yield AstT(nt.name)).getOrElse(BotT)
       case Many => AstT

src/main/scala/esmeta/analyzer/tychecker/TyChecker.scala

@@ -329,7 +329,7 @@ class TyChecker(
     List(view -> getCalleeState(AbsState.Empty, locals))
 
   /** initialization of ECMAScript environment */
-  lazy val init: Initialize = new Initialize(cfg)
+  lazy val init: Initialize = cfg.init
 
   /** global environment */
   lazy val base: Map[Global, AbsValue] = for {

src/main/scala/esmeta/cfg/CFG.scala

@@ -74,7 +74,7 @@ case class CFG(
     for {
       prod <- grammar.prods
       name = prod.name if !(grammar.lexicalNames contains name)
-      (rhs, idx) <- prod.rhsList.zipWithIndex
+      (rhs, idx) <- prod.rhsVec.zipWithIndex
       subIdx <- (0 until rhs.countSubs)
       to = (name, idx, subIdx)
     } rhs.getNts(subIdx) match

src/main/scala/esmeta/cfg/Node.scala

@@ -1,7 +1,7 @@
 package esmeta.cfg
 
 import esmeta.cfg.util.*
-import esmeta.util.{UId, Locational}
+import esmeta.util.{UId, Loc}
 import esmeta.ir.*
 import scala.collection.mutable.{Queue, ListBuffer}
 
@@ -29,6 +29,18 @@ sealed trait Node extends CFGElem with UId {
         case branch: Branch => add(branch.thenNode); add(branch.elseNode)
     }
     visited
+
+  /** get source locations */
+  def loc: Option[Loc] = this match
+    case block: Block =>
+      for {
+        head <- block.insts.headOption
+        headLoc <- head.loc
+        last <- block.insts.lastOption
+        lastLoc <- last.loc
+      } yield Loc(headLoc.start, lastLoc.end, headLoc.steps)
+    case call: Call     => call.callInst.loc
+    case branch: Branch => branch.cond.loc
 }
 
 /** block nodes */

src/main/scala/esmeta/cfg/util/JsonProtocol.scala

@@ -0,0 +1,30 @@
+package esmeta.cfg.util
+import esmeta.cfg.*
+import esmeta.util.*
+import io.circe.*, io.circe.syntax.*
+import io.circe.generic.auto.*, io.circe.generic.semiauto.*
+
+class JsonProtocol(cfg: CFG) extends BasicJsonProtocol {
+  // functions
+  given funcDecoder: Decoder[Func] = uidDecoder(cfg.funcMap.get)
+  given funcEncoder: Encoder[Func] = uidEncoder
+
+  // nodes
+  given nodeDecoder: Decoder[Node] = uidDecoder(cfg.nodeMap.get)
+  given nodeEncoder: Encoder[Node] = uidEncoder
+
+  // block nodes
+  given blockDecoder: Decoder[Block] =
+    uidDecoder(cfg.nodeMap.get(_).collect { case block: Block => block })
+  given blockEncoder: Encoder[Block] = uidEncoder
+
+  // call nodes
+  given callDecoder: Decoder[Call] =
+    uidDecoder(cfg.nodeMap.get(_).collect { case call: Call => call })
+  given callEncoder: Encoder[Call] = uidEncoder
+
+  // branch nodes
+  given branchDecoder: Decoder[Branch] =
+    uidDecoder(cfg.nodeMap.get(_).collect { case branch: Branch => branch })
+  given branchEncoder: Encoder[Branch] = uidEncoder
+}

src/main/scala/esmeta/compiler/Compiler.scala

@@ -839,7 +839,7 @@ class Compiler(
     case ProductionLiteral(lhsName, rhsName) =>
       getProductionData(lhsName, rhsName) match
         case Some((lhs, rhsIdx)) =>
-          ESyntactic(lhsName, lhs.params.map(_ => true), rhsIdx, Nil)
+          ESyntactic(lhsName, lhs.params.map(_ => true), rhsIdx, Vector.empty)
         case None =>
           EYet(lit.toString(true, false))
     case ErrorObjectLiteral(name) =>
@@ -1121,11 +1121,11 @@ class Compiler(
   /** production helpers */
   def getProductionData(lhsName: String, rhsName: String): Option[(Lhs, Int)] =
     val prod = grammar.nameMap(lhsName)
-    val rhsList = prod.rhsList.zipWithIndex.filter {
+    val rhsVec = prod.rhsVec.zipWithIndex.filter {
       case (rhs, _) if rhsName == "[empty]" => rhs.isEmpty
       case (rhs, _)                         => rhs.allNames contains rhsName
     }
-    rhsList match
+    rhsVec match
       case (rhs, idx) :: Nil => Some(prod.lhs, idx)
       case _                 => None
 

src/main/scala/esmeta/compiler/FuncBuilder.scala

@@ -35,7 +35,7 @@ case class FuncBuilder(
   /** bindings for nonterminals */
   var ntBindings: List[(String, Expr, Option[Int])] = algo.head match
     case SyntaxDirectedOperationHead(Some(target), _, _, _, _) =>
-      val rhs = grammar.nameMap(target.lhsName).rhsList(target.idx)
+      val rhs = grammar.nameMap(target.lhsName).rhsVec(target.idx)
       val rhsNames = rhs.nts.map(_.name)
       val rhsBindings = rhsNames.zipWithIndex.map {
         case (name, idx) => (name, ENAME_THIS, Some(idx))

src/main/scala/esmeta/error/UtilError.scala

@@ -8,3 +8,8 @@ case class InvalidGitVersion(msg: String)
 
 case class GitTagMismatch(hash: String, tagName: String)
   extends UtilError(s"Git tag mismatch: $hash != $tagName")
+
+case object NoGraalError extends UtilError("No Graal polyglot API")
+
+case class NoCommandError(command: String)
+  extends UtilError(s"Command not found: $command")