Merge branch 'master' into cup-bazel-rule

.bazelrc

@@ -0,0 +1,4 @@
+# https://errorprone.info/
+# TODO(#222) Increase to error when we build from jflex-1.7.1 with a lexer annotated with
+# @SuppressWarnings
+build --javacopt "-Xep:FallThrough:WARN"

.travis.bazelrc → .ci.bazelrc

@@ -1,3 +1,5 @@
+import %workspace%/.bazelrc
+
 # This is from Bazel's former travis setup, to avoid blowing up the RAM usage.
 startup --host_jvm_args=-Xms2000m
 startup --host_jvm_args=-Xmx3000m

.cirrus.yml

@@ -0,0 +1,12 @@
+# Copyright 2018 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+
+container:
+  image: cirrusci/bazel:latest
+task:
+  name: Bazel build and test
+  build_script:
+  - bazel --bazelrc=.ci.bazelrc info  --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST  release
+  - bazel --bazelrc=.ci.bazelrc build --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST  //...
+  test_script:
+  - bazel --bazelrc=.ci.bazelrc test  --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST  //...

.travis.yml

@@ -24,6 +24,7 @@ matrix:
     - scripts/test-unit.sh
     - scripts/mvn-site.sh
     - scripts/mvn-aggregate-srcs.sh
+    - scripts/preparare-deploy-source-code.sh
     after_success: scripts/send-code-coverage.sh
     env:
     - PUBLISH_SOURCES=true
@@ -133,4 +134,5 @@ deploy:
     - travis
     condition:
     - $PUBLISH_SOURCES
-  script: ./scripts/deploy-source-code.sh
+  script: scripts/deploy-aggregated-sources.sh
+

README.md

@@ -1,6 +1,9 @@
 <a href="https://travis-ci.org/jflex-de/jflex">
   <img alt="Build status" src="https://travis-ci.org/jflex-de/jflex.svg?branch=master" height="20">
 </a>
+<a href="https://cirrus-ci.com/github/jflex-de/jflex/master">
+  <img alt="Bazel build status" src="https://api.cirrus-ci.com/github/jflex-de/jflex.svg" height="20">
+</a>
 <a href="https://search.maven.org/artifact/de.jflex/jflex/">
   <img alt="Maven central" src="https://img.shields.io/maven-central/v/de.jflex/jflex.svg" height="20">
 </a>

docs/md/example.md

@@ -2,23 +2,31 @@ A simple Example: How to work with JFlex   {#Example}
 ========================================
 
 To demonstrate how a lexical specification with JFlex looks like, this
-section presents a part of the specification for the Java language. The
-example does not describe the whole lexical structure of Java programs,
-but only a small and simplified part of it (some keywords, some
-operators, comments and only two kinds of literals). It also shows how
-to interface with the LALR parser generator CUP [@CUP] and therefore uses
-a class `sym` (generated by CUP), where integer constants for the
-terminal tokens of the CUP grammar are declared. JFlex comes with a
-directory `examples`, where you can find a small standalone scanner that
-doesn’t need other tools like CUP to give you working example code without
-dependencies. 
-
-The `examples` directory also contains a *complete* JFlex specification of the
+section presents a part of the specification for the Java language.
+
+The example does not describe the whole lexical structure of Java programs,
+but only a small and simplified part of it:
+- some keywords,
+- some operators,
+- comments 
+- and only two kinds of literals.
+
+It also shows how to interface with the LALR parser generator CUP [@CUP] 
+and therefore uses a class `sym` (generated by CUP), where integer constants
+for the terminal tokens of the CUP grammar are declared. 
+
+You can find this example in `examples/cup-java-simplified`.
+
+The `examples/cup-java` directory also contains a *complete* JFlex specification of the
 lexical structure of Java programs together with the CUP parser specification
 for Java by C. Scott Ananian, obtained from the CUP [@CUP] web site (modified to
 interface with the JFlex scanner). Both specifications adhere to the Java
 Language Specification [@LangSpec].
 
+In `examples/standalone`, you can find a small standalone scanner that
+doesn’t need other dependencies or tools like CUP to give you working code. 
+
+
 ```
     /* JFlex example: partial Java language lexer specification */
     import java_cup.runtime.*;

jflex/examples/cup-java-minijava/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@jflex_rules//jflex:jflex.bzl", "jflex")
+load("//cup:cup.bzl", "cup")
+
+# The best practice is to define the rules in their respective directory in
+# - src/main/java/org/example/foo/BUILD
+# - src/main/jflex/BUILD
+# - src/test/java/org/example/foo/BUILD
+# - etc.
+# However, this example is simple enough and we can define all rules here.
+
+java_binary(
+    name = "minijava_bin",
+    main_class = "Yylex",
+    runtime_deps = [":minijava"],
+)
+
+java_library(
+    name = "minijava",
+    # glob is not a best practice, but it's good enough for this example
+    srcs = glob(["src/main/java/**/*.java"]) + [
+        ":gen_lexer",
+        ":gen_parser",
+    ],
+    deps = ["//cup:cup_runtime"],
+)
+
+jflex(
+    name = "gen_lexer",
+    srcs = ["src/main/jflex/minijava.flex"],
+    jflex_bin = "//jflex:jflex_bin",
+    outputs = ["Lexer.java"],
+)
+
+cup(
+    name = "gen_parser",
+    src = "src/main/cup/minijava.cup",
+    symbols = "sym",
+)
+
+# Tests
+java_test(
+    name = "LexerTest",
+    srcs = ["src/test/java/jflex/examples/minijava/LexerTest.java"],
+    deps = [
+        ":minijava",
+        "//cup/cup_runtime",
+        "//third_party/com/google/truth",
+    ],
+)

jflex/examples/cup-java-minijava/README.md

@@ -0,0 +1,32 @@
+# Example from the user manual
+
+This is the [example from the user manual](http://jflex.de/manual.html#Example).
+
+This example does not describe the whole lexical structure of Java programs, but only a small and
+simplified part of it (some keywords, some operators, comments and only two kinds of literals).
+It also shows how to interface with the LALR parser generator CUP and therefore uses a class `sym` 
+(generated by CUP), where integer constants for the terminal tokens of the CUP grammar are declared.
+
+For a full implementation of **Java 1.2**, see [cup-java](../cup-java).
+
+## Build, run, test
+
+### Using Maven
+
+    ../../mvnw package
+
+To run the parser:
+
+    java -cp target/cup-java-simplified-1.0.jar:../../../cup/cup/java-cup-11b.jar JavaParser <inputfile>
+
+or more simply the uberjar version:
+
+    java -jar target/cup-java-simplified-full-1.0.jar <inputfile>
+
+
+## Files
+
+* `src/main/jflex/minijava.flex`
+  Partial (simplified) specification of Java.
+* `src/test/java/jflex/examples/minijava/LexerTest.java`
+  Test of the generated lexer.

jflex/examples/cup-java-minijava/src/main/cup/minijava.cup

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 1998 C. Scott Ananian <[email protected]>
+ * Copyright (C) 1999 Gerwin Klein <[email protected]>
+ * Copyright (C) 2018 Google LLC
+ *
+ * This program is released under the terms of the GPL; see the file
+ * COPYING for more details.  There is NO WARRANTY on this code.
+ */
+package jflex.examples.minijava;
+
+import java_cup.runtime.*;
+
+
+/** Parser for a lamguages inspired by Java. */
+
+// Keywords
+terminal ABSTRACT; // "abstract"
+terminal BOOLEAN;  // "boolean" primitive_type
+terminal BREAK;    // "break"   break_statement
+
+// Identifier matches each string that starts with a character of class jletter followed by zero or
+// more characters of class jletterdigit
+terminal java.lang.String IDENTIFIER; // name
+
+// Literals
+terminal java.lang.Number INTEGER_LITERAL;
+terminal java.lang.String STRING_LITERAL;
+
+// Operators
+terminal EQ;
+terminal EQEQ; // equality_expression
+terminal PLUS;
+
+
+// 19.3) Lexical Structure
+non terminal literal;
+// 19.4) Types, Values, and Variables
+non terminal primitive_type;
+// 19.5) Names
+non terminal name;
+// 19.12) Expressions
+non terminal equality_expression;
+
+// Our simplified grammar
+non terminal goal;
+
+// TODO
+start with goal;
+
+goal ::=	name
+	;
+
+// 19.3) Lexical Structure.
+literal ::=	INTEGER_LITERAL
+	|	STRING_LITERAL
+	;
+
+// 19.4) Types, Values, and Variables
+type	::=	primitive_type
+	;
+primitive_type ::=
+	|	BOOLEAN
+	;
+
+// 19.5) Names
+name	::=	IDENTIFIER
+	;

testsuite/testcases/src/test/cases/manual-ex/manual.flex → jflex/examples/cup-java-minijava/src/main/jflex/minijava.flex

@@ -1,5 +1,10 @@
-/* JFlex example: part of Java language lexer specification */
-import java_cup.runtime.*;
+// JFlex example from the user Manual
+
+package jflex.examples.minijava;
+
+import java_cup.runtime.Symbol;
+
+/** Lexer of a very minimal version of the Java programming language. */
 
 %%
 

jflex/examples/cup-java-minijava/src/test/java/jflex/examples/minijava/LexerTest.java

@@ -0,0 +1,70 @@
+package jflex.examples.minijava;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Test for the generated {@link Lexer}.
+ *
+ * <p>The lexer is probably already correct thanks to the regression tests. This test class is
+ * mostly here to show how the lexer behaves.
+ */
+public class LexerTest {
+
+  private Lexer lexer;
+
+  @After
+  public void resetLexer() {
+    lexer = null;
+  }
+
+  @Test
+  public void scan_tokenIdentifier() throws IOException {
+    scan("helloWorld");
+    assertThat(nextToken()).isEqualTo(sym.IDENTIFIER);
+  }
+
+  @Test
+  public void scan_assignment() throws IOException {
+    scan("boolean debug = 2 == 1 + 1");
+    assertThat(nextToken()).isEqualTo(sym.BOOLEAN);
+    assertThat(nextToken()).isEqualTo(sym.IDENTIFIER);
+    assertThat(nextToken()).isEqualTo(sym.EQ);
+    assertThat(nextToken()).isEqualTo(sym.INTEGER_LITERAL);
+    assertThat(nextToken()).isEqualTo(sym.EQEQ);
+    assertThat(nextToken()).isEqualTo(sym.INTEGER_LITERAL);
+    assertThat(nextToken()).isEqualTo(sym.PLUS);
+    assertThat(nextToken()).isEqualTo(sym.INTEGER_LITERAL);
+    assertThat(nextToken()).isEqualTo(sym.EOF);
+  }
+
+  @SuppressWarnings("TryFailThrowable")
+  @Test
+  public void scan_illegalChar() throws IOException {
+    scan("boolean debug;");
+    assertThat(nextToken()).isEqualTo(sym.BOOLEAN);
+    assertThat(nextToken()).isEqualTo(sym.IDENTIFIER);
+    try {
+      nextToken();
+      fail("Character `;` is not declared in the minijava.flex");
+    } catch (Error expected) {
+      // This is bad, but the JFlex API doesn't allow better
+      // https://errorprone.info/bugpattern/TryFailThrowable
+    }
+  }
+
+  private void scan(String input) {
+    Reader in = new StringReader(input);
+    lexer = new Lexer(in);
+  }
+
+  private int nextToken() throws IOException {
+    return lexer.next_token().sym;
+  }
+}

jflex/src/main/java/jflex/Emitter.java

@@ -434,6 +434,9 @@ private void emitUserCode() {
   }
 
   private void emitClassName() {
+    // TODO(#222) Actually fix the fall-through violations
+    println("// See https://github.com/jflex-de/jflex/issues/222");
+    println("@SuppressWarnings(\"FallThrough\")");
     if (scanner.isPublic) print("public ");
 
     if (scanner.isAbstract) print("abstract ");

scripts/bazel.sh

@@ -9,7 +9,7 @@ source "$BASEDIR"/scripts/logger.sh
 set -e
 
 if [[ $TRAVIS ]]; then
-  BAZEL="bazel --bazelrc=$TRAVIS_BUILD_DIR/.travis.bazelrc --output_user_root=${HOME}/__bazel_travis_root__ --output_base=${HOME}/__bazel_output_base__"
+  BAZEL="bazel --bazelrc=$TRAVIS_BUILD_DIR/.ci.bazelrc --output_user_root=${HOME}/__bazel_travis_root__ --output_base=${HOME}/__bazel_output_base__"
 else
   BAZEL='bazel'
 fi

scripts/deploy-aggregated-sources.sh

@@ -0,0 +1,5 @@
+echo "Push to https://github.com/jflex-de/jflex/tree/aggregated-java-sources"
+cd repo
+# SECURITY NOTICE: Be sure to send stdout & stderr to /dev/null so that the the ${GITHUB_TOKEN} is$
+git remote set-url --push origin "https://${GITHUB_TOKEN}@github.com/jflex-de/jflex.git" > /dev/null
+git push