pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses´ this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  -->
<project xmlns="http://maven.apache.org/POM/4.0.0"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.cloudera.spark</groupId>
  <name>Cloudera Spark Cloud Integration Modules</name>
  <version>1.0-SNAPSHOT</version>
  <artifactId>cloudera-cloud-integration</artifactId>
  <packaging>pom</packaging>

  <description>
    Cloud integration for Apache Spark.
    This extends the built in packaging with tests and some other examples.
  </description>
  <url>https://github.com/hortonworks-spark/cloud-integration/</url>
  <licenses>
    <license>
      <name>Apache 2.0 License</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
      <distribution>repo</distribution>
    </license>
  </licenses>

  <modules>
    <module>cloud-examples</module>
    <module>spark-cloud-integration</module>
  </modules>

  <properties>
    <java.version>17</java.version>

    <scala.version>2.13.12</scala.version>
    <scala.binary.version>2.13</scala.binary.version>

    <!-- These examples use the version of Hadoop which Spark was built with-->
<!--
    <spark.version>3.0.0</spark.version>
-->
    <!--
    Maven is trying to hurt me today
    -->
<!--
    <spark.version>2.4.5.7.2.7.0-SNAPSHOT</spark.version>
    <spark.version>2.4.5.7.2.6.0-SNAPSHOT</spark.version>
    <spark.version>2.4.7.7.2.11.0-SNAPSHOT</spark.version>
    <scala.version>2.11.12</scala.version>
    <scala.binary.version>2.12</scala.binary.version>
    <spark.version>3.1.1.3.1.7270.0-SNAPSHOT</spark.version>

    -->

    <spark.version>4.0.0-SNAPSHOT</spark.version>

    <scala.version.tools>${scala.version}</scala.version.tools>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>


    <!--
     Spark shades its Guava use, so a version compatible with Hadoop
     is all that is needed here
    -->


    <guava.version>28.0-jre</guava.version>

<!--
    <scalatest.version>3.0.5</scalatest.version>
-->
    <scalatest.version>3.2.18</scalatest.version>
    <slf4j.version>2.0.11</slf4j.version>
    <log4j.version>2.22.1</log4j.version>

    <test.build.dir>${project.build.directory}/test-dir</test.build.dir>
    <test.build.data>${test.build.dir}</test.build.data>
    <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>

    <!-- plugin versions -->
    <maven-exec-plugin.version>1.4.0</maven-exec-plugin.version>
    <maven-surefire-plugin.version>3.2.5</maven-surefire-plugin.version>
    <!-- org.scala.tools-->
    <maven-scala-plugin.version>2.15.2</maven-scala-plugin.version>
    <!--net.alchim31.maven. fussy-->
    <scala-maven-plugin.version>4.7.1</scala-maven-plugin.version>
    <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
    <!-- override point: Hadoop XML file containing (directly or via XInclude references)
    the credentials needed to test against target cloud infrastructures. -->
    <unset>unset</unset>
    <cloud.test.configuration.file>${unset}</cloud.test.configuration.file>
    <log4j.debug>false</log4j.debug>
    <central.repo>https://repo1.maven.org/maven2</central.repo>

    <committer>${unset}</committer>

    <!-- set this to require a hadoop version-->
    <required.hadoop.version>${unset}</required.hadoop.version>

    <!-- are scale tests enabled ? -->
    <scale.test.enabled>unset</scale.test.enabled>
    <hive.tests.disabled>unset</hive.tests.disabled>

    <!-- Size in MB of huge files. -->
    <s3a.scale.test.huge.filesize>unset</s3a.scale.test.huge.filesize>
    <!-- Size in MB of the partion size in huge file uploads. -->
    <s3a.scale.test.huge.partitionsize>unset</s3a.scale.test.huge.partitionsize>
    <!-- Timeout in seconds for scale tests.-->
    <s3a.scale.test.timeout>3600</s3a.scale.test.timeout>


    <!-- if this flag is set, the loads _SUCCESS as JSON-->
    <s3a.committer.test.enabled>false</s3a.committer.test.enabled>

    <fs.s3a.committer.name>unset</fs.s3a.committer.name>
    <!-- JVM args for scalatest -->
    <scalatest.jvmargs>-ea</scalatest.jvmargs>


    <!-- plugins -->
    <!-- platform encoding override -->
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

    <!-- maven plugin versions -->
    <maven-deploy-plugin.version>2.8.1</maven-deploy-plugin.version>
    <maven-site-plugin.version>3.6</maven-site-plugin.version>
    <maven-stylus-skin.version>1.5</maven-stylus-skin.version>
    <maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
    <maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
    <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
    <maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
    <maven-javadoc-plugin.version>2.10.4</maven-javadoc-plugin.version>
    <maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
    <maven.properties.version>1.0-alpha-2</maven.properties.version>
    <maven-remote-resources-plugin.version>1.5</maven-remote-resources-plugin.version>
    <maven-resources-plugin.version>3.0.1</maven-resources-plugin.version>
    <apache-rat-plugin.version>0.10</apache-rat-plugin.version>
    <wagon-ssh.version>1.0</wagon-ssh.version>
    <maven-clover2-plugin.version>3.3.0</maven-clover2-plugin.version>
    <maven-bundle-plugin.version>2.5.0</maven-bundle-plugin.version>
    <lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
    <maven-checkstyle-plugin.version>2.17</maven-checkstyle-plugin.version>
    <checkstyle.version>7.5.1</checkstyle.version>
    <dependency-check-maven.version>1.4.3</dependency-check-maven.version>
    <mockito.version>2.23.4</mockito.version>
    <junit.version>4.13.1</junit.version>
    <scalacheck.version>1.17.0</scalacheck.version>

    <zinc.enabled>false</zinc.enabled>
  </properties>

  <repositories>
    <repository>
      <id>central</id>
      <!-- This must be at top, it makes maven try the central repository first
       and then others and hence faster dependency resolution -->
      <name>Maven Repository</name>
      <url>${central.repo}</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </repository>
  </repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>central</id>
      <url>${central.repo}</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </pluginRepository>
  </pluginRepositories>


  <!--
   The hadoop dependencies are explicitly declared
   to guarantee that the s3a committer code comes in,
   along with the associated changes to the hadoop-mapreduce
   JAR.

   This does force this POM to repeat the same exclusions
   of every Hadoop dependency as in spark-hadoop-cloud.

   -->
  <dependencyManagement>
    <dependencies>

      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <exclusions>
        </exclusions>
      </dependency>

      <!--Used for test classes -->
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <type>test-jar</type>
        <exclusions>
        </exclusions>
      </dependency>

      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
     </dependency>

      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <type>test-jar</type>
      </dependency>

      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <type>test-jar</type>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
      </dependency>

      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <type>test-jar</type>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-mllib_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-mllib-local_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
      </dependency>
            <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-avro_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
      </dependency>

      <!-- Junit is included as a compile-scoped dependency by jtransforms, which is
           a dependency of breeze. -->
<!--
      <dependency>
        <groupId>org.scalanlp</groupId>
        <artifactId>breeze_${scala.binary.version}</artifactId>
        <version>0.13.2</version>
        <exclusions>
          <exclusion>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-math3</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
-->
      <dependency>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest_${scala.binary.version}</artifactId>
        <version>${scalatest.version}</version>
      </dependency>
      <dependency>
        <groupId>org.mockito</groupId>
        <artifactId>mockito-core</artifactId>
        <version>${mockito.version}</version>
      </dependency>
      <dependency>
        <groupId>org.scalacheck</groupId>
        <artifactId>scalacheck_${scala.binary.version}</artifactId>
        <version>${scalacheck.version}</version>
      </dependency>
      <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>${junit.version}</version>
      </dependency>
      <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-api</artifactId>
        <version>${slf4j.version}</version>
      </dependency>
<!--
      <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-reload4j</artifactId>
        <version>${slf4j.version}</version>
      </dependency>
      <dependency>
        <groupId>ch.qos.reload4j</groupId>
        <artifactId>reload4j</artifactId>
        <version>${reload4j.version}</version>
        <exclusions>
          <exclusion>
            <groupId>com.sun.jdmk</groupId>
            <artifactId>jmxtools</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jmx</groupId>
            <artifactId>jmxri</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.mail</groupId>
            <artifactId>mail</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.jms</groupId>
            <artifactId>jmx</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.jms</groupId>
            <artifactId>jms</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
-->

      <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-slf4j2-impl</artifactId>
        <version>${log4j.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-api</artifactId>
        <version>${log4j.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-core</artifactId>
        <version>${log4j.version}</version>
      </dependency>
      <dependency>
        <!-- API bridge between log4j 1 and 2 -->
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-1.2-api</artifactId>
        <version>${log4j.version}</version>
        <scope>provided</scope>
      </dependency>

      <dependency>
        <groupId>com.google.guava</groupId>
        <artifactId>guava</artifactId>
        <version>${guava.version}</version>
      </dependency>

      <!-- figlet is here for some demo only -->
      <dependency>
        <groupId>com.github.lalyos</groupId>
        <artifactId>jfiglet</artifactId>
        <version>0.0.8</version>
      </dependency>


      <!-- use the shaded one for less version stress -->
      <!-- but exclude everything the connector forgets to-->
      <dependency>
        <groupId>com.google.cloud.bigdataoss</groupId>
        <artifactId>gcs-connector</artifactId>
        <version>3.0.0-SNAPSHOT</version>
        <classifier>shaded</classifier>
        <exclusions>
          <exclusion>
            <groupId>com.google.cloud.bigdataoss</groupId>
            <artifactId>gcsio</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.api-client</groupId>
            <artifactId>google-api-client-gson</artifactId>
          </exclusion>

          <exclusion>
            <groupId>com.google.oauth-client</groupId>
            <artifactId>google-oauth-client</artifactId>
          </exclusion>

          <exclusion>
            <groupId>com.google.cloud.bigdataoss</groupId>
            <artifactId>*</artifactId>
          </exclusion>

          <exclusion>
            <groupId>com.google.auto.value</groupId>
            <artifactId>auto-value-annotations</artifactId>
          </exclusion>

          <exclusion>
            <groupId>com.google.flogger</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
          </exclusion>

        </exclusions>
      </dependency>

<!--      <dependency>-->
<!--        <groupId>com.google.cloud.bigdataoss</groupId>-->
<!--        <artifactId>gcs-connector</artifactId>-->
<!--        <version>3.0.0-SNAPSHOT</version>-->
<!--      </dependency>-->

    </dependencies>

  </dependencyManagement>

  <build>
    <pluginManagement>
      <plugins>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-dependency-plugin</artifactId>
          <version>${maven-dependency-plugin.version}</version>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-enforcer-plugin</artifactId>
          <version>${maven-enforcer-plugin.version}</version>
          <configuration>
            <rules>
              <requireMavenVersion>
                <version>[3.6.1,)</version>
              </requireMavenVersion>
              <requireJavaVersion>
                <version>[1.8,)</version>
              </requireJavaVersion>
            </rules>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-assembly-plugin</artifactId>
          <version>${maven-assembly-plugin.version}</version>
        </plugin>

        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-compiler-plugin</artifactId>
          <version>3.6.1</version>
          <configuration>
            <source>${java.version}</source>
            <target>${java.version}</target>
            <useIncrementalCompilation>true</useIncrementalCompilation>
            <encoding>UTF-8</encoding>
            <maxmem>1024m</maxmem>
            <fork>true</fork>
            <compilerArgs>
              <arg>-Xlint:all,-serial,-path</arg>
            </compilerArgs>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-project-info-reports-plugin</artifactId>
          <version>2.9</version>
        </plugin>
<!--        liftedn from spark settin gs-->
        <plugin>
          <groupId>net.alchim31.maven</groupId>
          <artifactId>scala-maven-plugin</artifactId>
          <version>${scala-maven-plugin.version}</version>
          <executions>
            <execution>
              <id>eclipse-add-source</id>
              <goals>
                <goal>add-source</goal>
              </goals>
            </execution>
            <execution>
              <id>scala-compile-first</id>
              <goals>
                <goal>compile</goal>
              </goals>
            </execution>
            <execution>
              <id>scala-test-compile-first</id>
              <goals>
                <goal>testCompile</goal>
              </goals>
            </execution>
<!--            <execution>
              <id>attach-scaladocs</id>
              <phase>verify</phase>
              <goals>
                <goal>doc-jar</goal>
              </goals>
            </execution>-->
          </executions>

          <configuration>
            <scalaVersion>${scala.version}</scalaVersion>
            <recompileMode>incremental</recompileMode>
<!--            <useZincServer>${zinc.enabled}</useZincServer>-->

            <args>
              <arg>-unchecked</arg>
<!--              <arg>-deprecation</arg>-->
              <arg>-feature</arg>
              <arg>-explaintypes</arg>
<!--              <arg>-target:17</arg>-->
            </args>
            <jvmArgs>
              <jvmArg>-Xms256m</jvmArg>
              <jvmArg>-Xmx1024m</jvmArg>
              <!--
                          <jvmArg>-XX:PermSize=${PermGen}</jvmArg>
                          <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
                          <jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
              -->
            </jvmArgs>
            <javacArgs>
              <javacArg>-source</javacArg>
              <javacArg>${java.version}</javacArg>
              <javacArg>-target</javacArg>
              <javacArg>${java.version}</javacArg>
              <javacArg>-Xlint:all,-serial,-path</javacArg>
            </javacArgs>
          </configuration>
        </plugin>

        <!-- Scalatest runs all Scala tests -->
        <plugin>
          <groupId>org.scalatest</groupId>
          <artifactId>scalatest-maven-plugin</artifactId>
          <version>${scalatest-maven-plugin.version}</version>
          <!-- Note config is repeated in surefire config -->
          <configuration>
            <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
            <junitxml>.</junitxml>
            <filereports>SparkTestSuite.txt</filereports>
            <argLine>${scalatest.jvmargs}</argLine>
            <stderr/>
            <environmentVariables>
            </environmentVariables>
            <systemProperties>
              <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
              <derby.system.durability>test</derby.system.durability>
              <java.awt.headless>true</java.awt.headless>
              <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
              <spark.master.rest.enabled>false</spark.master.rest.enabled>
              <spark.ui.enabled>false</spark.ui.enabled>
              <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
              <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
              <spark.memory.debugFill>true</spark.memory.debugFill>

              <!-- Needed by sql/hive tests. -->
              <test.src.tables>src</test.src.tables>

              <cloud.test.configuration.file>${cloud.test.configuration.file}</cloud.test.configuration.file>
              <required.hadoop.version>${required.hadoop.version}</required.hadoop.version>
              <scale.test.enabled>${scale.test.enabled}</scale.test.enabled>

              <s3a.committer.test.enabled>${s3a.committer.test.enabled}</s3a.committer.test.enabled>
              <fs.s3a.committer.name>${fs.s3a.committer.name}</fs.s3a.committer.name>

              <hive.tests.disabled>${hive.tests.disabled}</hive.tests.disabled>
            </systemProperties>
          </configuration>
          <executions>
            <execution>
              <id>test</id>
              <goals>
                <goal>test</goal>
              </goals>
            </execution>
          </executions>
        </plugin>

        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-failsafe-plugin</artifactId>
          <version>${maven-surefire-plugin.version}</version>
        </plugin>
      </plugins>
    </pluginManagement>

    <plugins>
<!--
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-source-plugin</artifactId>
      </plugin>
-->
    </plugins>
  </build>
  <profiles>
    <!--
 This is a profile to enable the use of the ASF snapshot and staging repositories
 during a build. It is useful when testing againt nightly or RC releases of dependencies.
 It MUST NOT be used when building copies of Spark to use in production of for distribution,
 -->
    <profile>
      <id>staging</id>
      <properties>
        <!-- override point for ASF staging/snapshot repos -->
        <asf.staging>https://repository.apache.org/content/groups/staging/</asf.staging>
        <asf.snapshots>https://repository.apache.org/content/repositories/snapshots/</asf.snapshots>
      </properties>

      <pluginRepositories>
        <pluginRepository>
          <id>ASF Staging</id>
          <url>${asf.staging}</url>
        </pluginRepository>
        <pluginRepository>
          <id>ASF Snapshots</id>
          <url>${asf.snapshots}</url>
          <snapshots>
            <enabled>true</enabled>
          </snapshots>
          <releases>
            <enabled>false</enabled>
          </releases>
        </pluginRepository>
      </pluginRepositories>

      <repositories>
        <repository>
          <id>ASF Staging</id>
          <url>${asf.staging}</url>
        </repository>
        <repository>
          <id>ASF Snapshots</id>
          <url>${asf.snapshots}</url>
          <snapshots>
            <enabled>true</enabled>
          </snapshots>
          <releases>
            <enabled>false</enabled>
          </releases>
        </repository>
      </repositories>
    </profile>

<!--
    <profile>
      <id>spark-2.3</id>
      <properties>
        &lt;!&ndash;<required.hadoop.version>3.0.0-alpha4-SNAPSHOT</required.hadoop.version>&ndash;&gt;
        <required.hadoop.version>2.11</required.hadoop.version>
        <guava.version>11.0.2</guava.version>
        <spark.version>2.3.0</spark.version>
      </properties>
    </profile>
-->

    <profile>
      <id>scale</id>
      <activation>
        <property>
          <name>scale</name>
        </property>
      </activation>
      <properties>
        <scale.test.enabled>true</scale.test.enabled>
      </properties>
    </profile>


    <!--turn off the hive tests-->
    <profile>
      <id>skipHiveTests</id>
      <activation>
        <property>
          <name>skipHiveTests</name>
        </property>
      </activation>
      <properties>
        <hive.tests.disabled>true</hive.tests.disabled>
      </properties>
    </profile>

    <profile>
      <id>staging-committer</id>
      <properties>
        <fs.s3a.committer.enabled>true</fs.s3a.committer.enabled>
        <fs.s3a.committer.name>staging</fs.s3a.committer.name>

        <fs.s3a.committer.test.enabled>true</fs.s3a.committer.test.enabled>
      </properties>
    </profile>


    <profile>
      <id>directory-committer</id>
      <properties>
        <fs.s3a.committer.enabled>true</fs.s3a.committer.enabled>
        <fs.s3a.committer.test.enabled>true</fs.s3a.committer.test.enabled>
        <fs.s3a.committer.name>directory</fs.s3a.committer.name>
      </properties>
    </profile>

    <profile>
      <id>partition-committer</id>
      <properties>
        <fs.s3a.committer.enabled>true</fs.s3a.committer.enabled>
        <fs.s3a.committer.test.enabled>true</fs.s3a.committer.test.enabled>
        <fs.s3a.committer.name>partitioned</fs.s3a.committer.name>
      </properties>
    </profile>

    <!-- this just turns off things -->
    <profile>
      <id>uncommitted</id>
      <activation>
        <property>
          <name>uncommitted</name>
        </property>
      </activation>
      <properties>
        <fs.s3a.committer.enabled>true</fs.s3a.committer.enabled>
        <fs.s3a.committer.test.enabled>true</fs.s3a.committer.test.enabled>
        <fs.s3a.committer.name>unset</fs.s3a.committer.name>
      </properties>
    </profile>

    <profile>
      <id>magic-committer</id>
      <activation><activeByDefault>true</activeByDefault></activation>
      <properties>
        <fs.s3a.committer.enabled>true</fs.s3a.committer.enabled>
        <s3a.committer.test.enabled>true</s3a.committer.test.enabled>
        <fs.s3a.committer.name>magic</fs.s3a.committer.name>
      </properties>
    </profile>

    <!--
     Switch scalatest to JVM debug.
     This blocks the process and lets you attach
      -->
    <profile>
      <id>debug</id>
      <activation>
        <property>
          <name>debug</name>
        </property>
      </activation>
      <properties>
        <!-- JVM args for scalatest -->
        <scalatest.jvmargs>-ea -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005</scalatest.jvmargs>
<!--
        <scalatest.jvmargs>-ea -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,address=4000,server=y,suspend=n</scalatest.jvmargs>
-->
      </properties>
    </profile>


    <profile>
      <id>spark-master</id>
      <activation>
        <property>
          <name>spark-master</name>
        </property>
      </activation>
      <properties>
        <spark.version>4.0.0-SNAPSHOT</spark.version>
      </properties>
    </profile>

    <profile>
      <id>hadoop-trunk</id>
      <activation>
        <property>
          <name>hadoop-trunk</name>
        </property>
      </activation>
      <properties>
        <hadoop.version>3.5.0-SNAPSHOT</hadoop.version>
      </properties>
    </profile>

    <profile>
      <id>hadoop-3.4</id>
      <properties>
        <hadoop.version>3.4.0</hadoop.version>
      </properties>
    </profile>

    <profile>
      <id>hadoop-3.3.5</id>
      <properties>
        <hadoop.version>3.3.5</hadoop.version>
      </properties>
    </profile>

    <profile>
      <id>hadoop-branch-3.3</id>
      <properties>
        <hadoop.version>3.3.9-SNAPSHOT</hadoop.version>
      </properties>
    </profile>

    <!-- switch to spark cdpd-master. Doesn't change hadoop settings -->
    <profile>
      <id>spark-cdpd-3</id>
      <properties>
        <spark.version>3.3.0.7.2.17.0-SNAPSHOT</spark.version>
      </properties>
    </profile>


    <profile>
      <id>oldguava</id>
      <activation>
        <property>
          <name>oldguava</name>
        </property>
      </activation>
      <properties>
        <guava.version>19.0</guava.version>
      </properties>
    </profile>

  </profiles>


</project>