dynamically create BigQuery metrics view

Closes #392

Author: Aaron Meihm

Diff for: bin/generate_meta.sh

@@ -7,5 +7,5 @@
 f=`mktemp`
 outpath=contrib/common/alertmeta.go
 
-java com.mozilla.secops.alert.AlertMeta "${f}"
+java com.mozilla.secops.alert.AlertMeta gometa "${f}"
 gofmt $f > $outpath

Diff for: bin/generate_view.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Generate BigQuery metrics view
+#
+# Called by maven and should not be run directly
+
+f=./target/metrics-view.sql
+java com.mozilla.secops.alert.AlertMeta metricsview "${f}"
+cat $f

Diff for: pom.xml

@@ -76,10 +76,22 @@
                         <goals>
                             <goal>exec</goal>
                         </goals>
+                        <configuration>
+                            <executable>bin/generate_meta.sh</executable>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>generate-metrics-view</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <executable>bin/generate_view.sh</executable>
+                        </configuration>
                     </execution>
                 </executions>
                 <configuration>
-                    <executable>bin/generate_meta.sh</executable>
                     <environmentVariables>
                         <CLASSPATH>
                             ${maven.compile.classpath}:target/classes

Diff for: src/main/java/com/mozilla/secops/alert/AlertMeta.java

@@ -6,6 +6,7 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.ArrayList;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -199,6 +200,15 @@ public String getKey() {
       return key;
     }
 
+    /**
+     * Return if key is considered sensitive
+     *
+     * @return boolean
+     */
+    public boolean getIsSensitive() {
+      return isSensitive;
+    }
+
     /**
      * Obtain given associated key type
      *
@@ -324,17 +334,38 @@ public AlertMeta(@JsonProperty("key") String key, @JsonProperty("value") String
   }
 
   /**
-   * Convert metadata to Golang
+   * Convert metadata to various formats
+   *
+   * <p>This main function can be used to export metadata keys in golang format, and to export a
+   * BigQuery view query that is suitable for metrics usage.
    *
    * @param args Arguments
    */
   public static void main(String[] args) throws IOException {
-    BufferedWriter w = new BufferedWriter(new FileWriter(args[0]));
-    w.write("package common\n\n// This file is automatically generated.\n\nconst (\n");
-    for (Key k : Key.values()) {
-      w.write(String.format("META_%s = \"%s\"\n", k.name(), k.getKey()));
+    if (args[0].equals("gometa")) {
+      BufferedWriter w = new BufferedWriter(new FileWriter(args[1]));
+      w.write("package common\n\n// This file is automatically generated.\n\nconst (\n");
+      for (Key k : Key.values()) {
+        w.write(String.format("META_%s = \"%s\"\n", k.name(), k.getKey()));
+      }
+      w.write(")\n");
+      w.close();
+    } else if (args[0].equals("metricsview")) {
+      BufferedWriter w = new BufferedWriter(new FileWriter(args[1]));
+      ArrayList<String> sFields = new ArrayList<>();
+      for (Key k : Key.values()) {
+        if (k.getIsSensitive()) {
+          sFields.add("'" + k.getKey() + "'");
+        }
+      }
+      w.write(
+          String.format(
+              "SELECT EXTRACT(DATETIME FROM _PARTITIONTIME) AS partitiontime, "
+                  + "id, timestamp, severity, category, ARRAY(\nSELECT AS STRUCT key, "
+                  + "value FROM UNNEST(metadata) WHERE\nKEY NOT IN (%s)\n) AS metadata\n"
+                  + "FROM <<table>>\n",
+              String.join(", ", sFields)));
+      w.close();
     }
-    w.write(")\n");
-    w.close();
   }
 }