Add Linear Regression to Aggregator

.classpath

@@ -5,18 +5,15 @@
 	<classpathentry kind="lib" path="rtlib/scannotation-1.0.3.jar"/>
 	<classpathentry kind="lib" path="rtlib/ST-4.0.8.jar"/>
 	<classpathentry kind="lib" path="lib/antlr-4.5-complete.jar"/>
-	<classpathentry kind="lib" path="lib/commons-lang-2.4.jar"/>
 	<classpathentry kind="lib" path="lib/commons-math-2.1.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-core-1.0.4.jar"/>
 	<classpathentry kind="lib" path="lib/hamcrest-all-1.3.jar"/>
 	<classpathentry kind="lib" path="lib/hamcrest-core-1.3.jar"/>
 	<classpathentry kind="lib" path="lib/junit-4.11.jar"/>
 	<classpathentry kind="lib" path="lib/mrunit-0.9.0-incubating-hadoop1.jar"/>
-	<classpathentry kind="lib" path="lib/datagen/commons-io-2.4.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/cssparser-0.9.23.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/dom4j-2.0.2.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/eclipse.jdt.core.jar"/>
-	<classpathentry kind="lib" path="lib/datagen/gson-2.7.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/jsoup-1.10.3.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/maven-model-3.0.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/org.eclipse.core.contenttype_3.4.100.dist.jar"/>
@@ -63,8 +60,6 @@
 	<classpathentry kind="lib" path="lib/datagen/org.eclipse.text-3.5.101-sources.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/org.eclipse.text-3.5.101.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/protobuf-java-2.5.0.jar"/>
-	<classpathentry kind="lib" path="lib/datagen/slf4j-api-1.4.3.jar"/>
-	<classpathentry kind="lib" path="lib/datagen/slf4j-log4j12-1.4.3.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/svnkit-1.7.6.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/xercesImpl.jar"/>
 	<classpathentry kind="lib" path="lib/datagen/xml-apis.jar"/>
@@ -74,7 +69,61 @@
 	<classpathentry kind="lib" path="lib/evaluator/com.springsource.org.codehaus.jackson.mapper-1.4.2.jar"/>
 	<classpathentry kind="lib" path="lib/evaluator/commons-configuration-1.6.jar"/>
 	<classpathentry kind="lib" path="lib/evaluator/commons-httpclient-3.1.0.redhat-7.jar"/>
-	<classpathentry kind="lib" path="lib/evaluator/commons-io-2.4.jar"/>
 	<classpathentry kind="lib" path="compile"/>
+	<classpathentry kind="lib" path="lib/weka/arpack_combined.jar"/>
+	<classpathentry kind="lib" path="lib/weka/core.jar"/>
+	<classpathentry kind="lib" path="lib/weka/mtj.jar"/>
+	<classpathentry kind="lib" path="lib/weka/weka.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/commons-codec-1.10.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/commons-compress-1.18.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/commons-io-2.5.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/commons-lang3-3.6.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/commons-math3-3.5.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/deeplearning4j-common-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/deeplearning4j-nlp-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/deeplearning4j-nn-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/gson-2.8.0.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/guava-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/jackson-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-1.5.3.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-platform-1.5.3.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacv-1.5.3.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-api-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-common-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-native-1.0.0-beta7-linux-armhf.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-native-1.0.0-beta7-linux-ppc64le.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-native-1.0.0-beta7-linux-x86_64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-native-1.0.0-beta7-macosx-x86_64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-native-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-native-api-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nd4j-native-platform-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/nearestneighbor-core-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/neoitertools-1.0.0.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-0.3.9-1.5.3-linux-arm64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-0.3.9-1.5.3-linux-armhf.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-0.3.9-1.5.3-linux-ppc64le.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-0.3.9-1.5.3-linux-x86_64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-0.3.9-1.5.3-linux-x86.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-0.3.9-1.5.3-macosx-x86_64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-0.3.9-1.5.3.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/openblas-platform-0.3.9-1.5.3.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/opencsv-2.3.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/opencv-platform-4.3.0-1.5.3.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/protobuf-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/slf4j-api-1.7.21.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/slf4j-simple-1.7.21.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/threadly-4.10.0.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-1.5.3-linux-arm64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-1.5.3-linux-armhf.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-1.5.3-linux-ppc64le.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-1.5.3-linux-x86_64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-1.5.3-linux-x86.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/javacpp-1.5.3-macosx-x86_64.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/jna-4.3.0.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/jna-platform-4.3.0.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/deeplearning4j-core-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/deeplearning4j-tsne-1.0.0-beta7.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/flatbuffers-java-1.10.0.jar"/>
+	<classpathentry kind="lib" path="lib/dl4j/commons-lang-2.6.jar"/>
 	<classpathentry kind="output" path="build/classes"/>
 </classpath>

src/antlr/Boa.g

@@ -112,6 +112,7 @@ type returns [AbstractType ast]
 	| q=queueType    { $ast = $q.ast; }
 	| set=setType    { $ast = $set.ast; }
 	| e=enumType     { $ast = $e.ast; }
+	| model=modelType { $ast = $model.ast; }
 	| id=identifier  { $ast = $id.ast; }
 	;
 
@@ -202,7 +203,12 @@ outputType returns [OutputType ast]
 	@after { $ast.setPositions($l, $c, getEndLine(), getEndColumn()); }
 	: OUTPUT (tk=SET { $ast = new OutputType((Identifier)new Identifier($tk.text).setPositions(getStartLine($tk), getStartColumn($tk), getEndLine($tk), getEndColumn($tk))); } | id=identifier { $ast = new OutputType($id.ast); }) (LPAREN el=expressionList RPAREN { $ast.setArgs($el.list); })? (LBRACKET m=component RBRACKET { $ast.addIndice($m.ast); })* OF m=component { $ast.setType($m.ast); } (WEIGHT m=component { $ast.setWeight($m.ast); })? (FORMAT LPAREN el=expressionList RPAREN)?
 	;
-
+modelType returns [ModelType ast]
+	locals [int l, int c]
+	@init { $l = getStartLine(); $c = getStartColumn(); }
+	@after { $ast.setPositions($l, $c, getEndLine(), getEndColumn()); }
+	: MODEL OF id=identifier { $ast = new ModelType($id.ast); } OF m=component { $ast.setType($m.ast); }
+	;
 functionType returns [FunctionType ast]
 	locals [int l, int c]
 	@init {
@@ -610,6 +616,7 @@ identifier returns [Identifier ast]
 	| lit=MAP      { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=STACK    { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=QUEUE    { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
+	| lit=MODEL    { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=SET      { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=FOR      { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=FOREACH  { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
@@ -690,6 +697,7 @@ DO        : 'do';
 MAP       : 'map';
 STACK     : 'stack';
 QUEUE     : 'queue';
+MODEL     : 'model';
 SET       : 'set';
 FOR       : 'for';
 FOREACH   : 'foreach';

src/antlr/BoaNoActions.g

@@ -55,6 +55,7 @@ type
 	| queueType
 	| setType
 	| enumType
+	| modelType
 	| identifier
 	;
 
@@ -103,6 +104,10 @@ setType
 outputType
 	: OUTPUT (SET | identifier) (LPAREN expressionList RPAREN)? (LBRACKET component RBRACKET)* OF component (WEIGHT component)? (FORMAT LPAREN expressionList RPAREN)?
 	;
+
+modelType
+	: MODEL of identifier OF component
+	;
 
 functionType
 	: FUNCTION LPAREN (identifier COLON type (COMMA identifier COLON type)*)? RPAREN (COLON type)?
@@ -352,6 +357,7 @@ identifier
 	| lit=MAP      { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=STACK    { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=QUEUE    { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
+	| lit=MODEL    { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=SET      { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=FOR      { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
 	| lit=FOREACH  { notifyErrorListeners("keyword '" + $lit.text + "' can not be used as an identifier"); }
@@ -417,6 +423,7 @@ DO        : 'do';
 MAP       : 'map';
 STACK     : 'stack';
 QUEUE     : 'queue';
+MODEL     : 'model';
 SET       : 'set';
 FOR       : 'for';
 FOREACH   : 'foreach';

src/java/boa/aggregators/Aggregator.java

@@ -25,6 +25,7 @@
 import boa.functions.BoaCasts;
 import boa.io.EmitKey;
 import boa.io.EmitValue;
+import boa.runtime.Tuple;
 
 /**
  * The base class for all Boa aggregators.

src/java/boa/aggregators/ml/AdaBoostM1Aggregator.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2014, Hridesh Rajan, Robert Dyer,
+ *                 and Iowa State University of Science and Technology
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package boa.aggregators.ml;
+
+import boa.runtime.Tuple;
+import boa.aggregators.AggregatorSpec;
+import weka.classifiers.meta.AdaBoostM1;
+import java.io.IOException;
+
+/**
+ * A Boa aggregator for training the model using AdaBoostM1.
+ *
+ * @author ankuraga
+ * @author nmtiwari
+ */
+@AggregatorSpec(name = "adaboost", formalParameters = { "string" })
+public class AdaBoostM1Aggregator extends MLAggregator {
+	private AdaBoostM1 model;
+
+	public AdaBoostM1Aggregator() {
+	}
+
+	public AdaBoostM1Aggregator(final String s) {
+		super(s);
+	}
+
+	@Override
+	public void aggregate(final String[] data, String metadata) throws IOException, InterruptedException {
+		aggregate(data, metadata, "AdaBoostM1");
+	}
+
+	@Override
+	public void aggregate(final Tuple data, final String metadata) throws IOException, InterruptedException {
+		aggregate(data, metadata, "AdaBoostM1");
+	}
+
+	@Override
+	public void aggregate(String data, String metadata) throws IOException, InterruptedException {
+	}
+
+	@Override
+	public void finish() throws IOException, InterruptedException {
+		try {
+			this.model = new AdaBoostM1();
+			this.model.setOptions(options);
+			this.model.buildClassifier(this.trainingSet);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		this.saveModel(this.model);
+		this.evaluate(this.model, this.trainingSet);
+		this.evaluate(this.model, this.testingSet);
+		this.collect(this.model.toString());
+	}
+
+}

src/java/boa/aggregators/ml/AdditiveRegressionAggregator.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2014, Hridesh Rajan, Robert Dyer,
+ *                 and Iowa State University of Science and Technology
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package boa.aggregators.ml;
+
+import boa.runtime.Tuple;
+import boa.aggregators.AggregatorSpec;
+import weka.classifiers.meta.AdditiveRegression;
+import java.io.IOException;
+
+/**
+ * A Boa aggregator for training the model using AdditiveRegression.
+ *
+ * @author ankuraga/
+ * @author nmtiwari
+ */
+@AggregatorSpec(name = "additiveregression", formalParameters = { "string" })
+public class AdditiveRegressionAggregator extends MLAggregator {
+	private AdditiveRegression model;
+
+	public AdditiveRegressionAggregator() {
+	}
+
+	public AdditiveRegressionAggregator(final String s) {
+		super(s);
+	}
+
+	public void aggregate(String[] data, String metadata) throws IOException, InterruptedException {
+		aggregate(data, metadata, "AdditiveRegression");
+	}
+
+	public void aggregate(final Tuple data, final String metadata) throws IOException, InterruptedException {
+		aggregate(data, metadata, "AdditiveRegression");
+	}
+
+	@Override
+	public void aggregate(String data, String metadata) throws IOException, InterruptedException {
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void finish() throws IOException, InterruptedException {
+		try {
+			this.model = new AdditiveRegression();
+			this.model.setOptions(options);
+			this.model.buildClassifier(this.trainingSet);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		this.saveModel(this.model);
+		this.evaluate(this.model, this.trainingSet);
+		this.evaluate(this.model, this.testingSet);
+		this.collect(this.model.toString());
+	}
+}

src/java/boa/aggregators/ml/AttributeSelectedClassifierAggregator.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2014, Hridesh Rajan, Robert Dyer,
+ *                 and Iowa State University of Science and Technology
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package boa.aggregators.ml;
+
+import boa.runtime.Tuple;
+import boa.aggregators.AggregatorSpec;
+import weka.classifiers.meta.AttributeSelectedClassifier;
+
+import java.io.IOException;
+
+/**
+ * A Boa aggregator for training the model using AttributeSelectedClassifier.
+ *
+ * @author ankuraga
+ */
+@AggregatorSpec(name = "attributeselectedclassifier", formalParameters = { "string" })
+public class AttributeSelectedClassifierAggregator extends MLAggregator {
+	private AttributeSelectedClassifier model;
+
+	public AttributeSelectedClassifierAggregator() {
+	}
+
+	public AttributeSelectedClassifierAggregator(final String s) {
+		super(s);
+	}
+
+	public void aggregate(String[] data, final String metadata) throws IOException, InterruptedException {
+		aggregate(data, metadata, "AttributeSelectedClassifier");
+	}
+
+	public void aggregate(final Tuple data, final String metadata) throws IOException, InterruptedException {
+		aggregate(data, metadata, "AttributeSelectedClassifier");
+	}
+
+	@Override
+	public void aggregate(String data, String metadata) throws IOException, InterruptedException {
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void finish() throws IOException, InterruptedException {
+		try {
+			this.model = new AttributeSelectedClassifier();
+			this.model.setOptions(options);
+			this.model.buildClassifier(this.trainingSet);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		this.saveModel(this.model);
+		this.evaluate(this.model, this.trainingSet);
+		this.evaluate(this.model, this.testingSet);
+		this.collect(this.model.toString());
+	}
+}