Ability to Use Jar as a Library Bundled in Other Tools (#113)

* Changes and improvements to make this jar make sense to be callable as a library. * Delay initial report or progress. * Expose duration via MirrorStats * Up version number * Created separate profile for uberjar so default can be used for regular jar
cobbzilla · Jan 25, 2021 · 65dca48 · 65dca48
1 parent ad91232
commit 65dca48
Show file tree

Hide file tree

Showing 20 changed files with 224 additions and 72 deletions.
diff --git a/pom.xml b/pom.xml
@@ -11,7 +11,7 @@
 
     <groupId>org.cobbzilla</groupId>
     <artifactId>s3s3mirror</artifactId>
-    <version>2.1.4-SNAPSHOT</version>
+    <version>2.1.5-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <licenses>
@@ -107,7 +107,7 @@
 
     <build>
         <plugins>
-            <!-- Force Java 1.7 at all times -->
+            <!-- Force Java 1.8 at all times -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
@@ -118,41 +118,55 @@
                     <showWarnings>true</showWarnings>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <version>2.1</version>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>org.cobbzilla.s3s3mirror.MirrorMain</mainClass>
-                                </transformer>
-                            </transformers>
-                            <!-- Exclude signed jars to avoid errors
-                            see: http://stackoverflow.com/a/6743609/1251543
-                            -->
-                            <filters>
-                                <filter>
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>META-INF/*.SF</exclude>
-                                        <exclude>META-INF/*.DSA</exclude>
-                                        <exclude>META-INF/*.RSA</exclude>
-                                    </excludes>
-                                </filter>
-                            </filters>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
         </plugins>
     </build>
 
+    <profiles>
+        <profile>
+            <id>base</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+        </profile>
+        <profile>
+            <id>uberjar</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-shade-plugin</artifactId>
+                        <version>2.1</version>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>shade</goal>
+                                </goals>
+                                <configuration>
+                                    <transformers>
+                                        <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                            <mainClass>org.cobbzilla.s3s3mirror.MirrorMain</mainClass>
+                                        </transformer>
+                                    </transformers>
+                                    <!-- Exclude signed jars to avoid errors
+                                    see: http://stackoverflow.com/a/6743609/1251543
+                                    -->
+                                    <filters>
+                                        <filter>
+                                            <artifact>*:*</artifact>
+                                            <excludes>
+                                                <exclude>META-INF/*.SF</exclude>
+                                                <exclude>META-INF/*.DSA</exclude>
+                                                <exclude>META-INF/*.RSA</exclude>
+                                            </excludes>
+                                        </filter>
+                                    </filters>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>
diff --git a/s3s3mirror.bat b/s3s3mirror.bat
@@ -1,2 +1,2 @@
 @echo off
-java -Ds3s3mirror.version=2.1.4 -jar target/s3s3mirror-2.1.4-SNAPSHOT.jar %*
+java -Ds3s3mirror.version=2.1.5 -jar target/s3s3mirror-2.1.4-SNAPSHOT.jar %*
diff --git a/s3s3mirror.sh b/s3s3mirror.sh
@@ -3,7 +3,7 @@
 
 THISDIR=$(cd "$(dirname $0)" && pwd)
 
-VERSION=2.1.4
+VERSION=2.1.5
 JARFILE="${THISDIR}/target/s3s3mirror-${VERSION}-SNAPSHOT.jar"
 VERSION_ARG="-Ds3s3mirror.version=${VERSION}"
 

diff --git a/src/main/java/org/cobbzilla/s3s3mirror/BucketMirrorMaker.java b/src/main/java/org/cobbzilla/s3s3mirror/BucketMirrorMaker.java
@@ -0,0 +1,17 @@
+package org.cobbzilla.s3s3mirror;
+
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import org.cobbzilla.s3s3mirror.stats.MirrorStats;
+
+public class BucketMirrorMaker {
+
+    public MirrorStats makeCopy(MirrorOptions options) {
+        options.initDerivedFields();
+        options.setAwsCredentialProviders(new DefaultAWSCredentialsProviderChain());
+
+        MirrorContext context = new MirrorContext(options);
+        MirrorMaster mirrorMaster = new MirrorMaster(context);
+
+        return mirrorMaster.mirror();
+    }
+}
diff --git a/src/main/java/org/cobbzilla/s3s3mirror/KeyCopyJob.java b/src/main/java/org/cobbzilla/s3s3mirror/KeyCopyJob.java
@@ -67,12 +67,12 @@ public void run() {
                     if (options.isVerbose()) getLog().info("successfully copied "+key+" -> "+getKeyDestination());
                     context.getStats().objectsCopied.incrementAndGet();
                 } else {
-                    context.getStats().copyErrors.incrementAndGet();
+                    context.getStats().addErroredCopy(this);
                 }
             }
         } catch (Exception e) {
             getLog().error("error copying key: " + key + ": " + e);
-            if (!options.isDryRun()) context.getStats().copyErrors.incrementAndGet();
+            if (!options.isDryRun()) context.getStats().addErroredCopy(this);
 
         } finally {
             synchronized (notifyLock) {
@@ -138,4 +138,14 @@ private boolean shouldTransfer() throws Exception {
 
         return comparisonStrategy.sourceDifferent(summary, destination);
     }
+
+    @Override
+    public String getSource() {
+        return summary.getKey();
+    }
+
+    @Override
+    public String getDestination() {
+        return keyDestination;
+    }
 }
diff --git a/src/main/java/org/cobbzilla/s3s3mirror/KeyDeleteJob.java b/src/main/java/org/cobbzilla/s3s3mirror/KeyDeleteJob.java
@@ -1,6 +1,7 @@
 package org.cobbzilla.s3s3mirror;
 
 import com.amazonaws.services.s3.AmazonS3Client;
+import org.cobbzilla.s3s3mirror.stats.MirrorStats;
 import org.cobbzilla.s3s3mirror.comparisonstrategies.strategies.NoImplementationComparisonStrategy;
 import org.cobbzilla.s3s3mirror.store.FileSummary;
 
@@ -57,13 +58,13 @@ public void run() {
                 if (deletedOK) {
                     stats.objectsDeleted.incrementAndGet();
                 } else {
-                    stats.deleteErrors.incrementAndGet();
+                    stats.addErroredDelete(this);
                 }
             }
 
         } catch (Exception e) {
             getLog().error("error deleting key: "+key+": "+e);
-            if (!options.isDryRun()) context.getStats().deleteErrors.incrementAndGet();
+            if (!options.isDryRun()) context.getStats().addErroredDelete(this);
 
         } finally {
             synchronized (notifyLock) {

diff --git a/src/main/java/org/cobbzilla/s3s3mirror/KeyJob.java b/src/main/java/org/cobbzilla/s3s3mirror/KeyJob.java
@@ -1,5 +1,6 @@
 package org.cobbzilla.s3s3mirror;
 
 public interface KeyJob extends Runnable {
-
+    String getSource();
+    String getDestination();
 }
diff --git a/src/main/java/org/cobbzilla/s3s3mirror/MirrorContext.java b/src/main/java/org/cobbzilla/s3s3mirror/MirrorContext.java
@@ -3,6 +3,7 @@
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
+import org.cobbzilla.s3s3mirror.stats.MirrorStats;
 
 @AllArgsConstructor
 public class MirrorContext {

diff --git a/src/main/java/org/cobbzilla/s3s3mirror/MirrorMaster.java b/src/main/java/org/cobbzilla/s3s3mirror/MirrorMaster.java
@@ -1,6 +1,7 @@
 package org.cobbzilla.s3s3mirror;
 
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.s3s3mirror.stats.MirrorStats;
 
 import java.util.concurrent.*;
 
@@ -16,11 +17,12 @@ public class MirrorMaster {
 
     public MirrorMaster(MirrorContext context) { this.context = context; }
 
-    public void mirror() {
-
+    public MirrorStats mirror() {
+        ScheduledExecutorService scheduledExecutorService = null;
         log.info("version "+VERSION+" starting");
 
         final MirrorOptions options = context.getOptions();
+        context.getStats().logFailedOperationInfo.set(options.isDetailedErrorLogging());
 
         if (options.isVerbose() && options.hasCtime()) log.info("will not copy anything older than "+options.getCtime()+" (cutoff="+options.getMaxAgeDate()+")");
 
@@ -35,6 +37,15 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
 
         final ThreadPoolExecutor executorService = new ThreadPoolExecutor(options.getMaxThreads(), options.getMaxThreads(), 1, TimeUnit.MINUTES, workQueue, rejectedExecutionHandler);
 
+        if (context.getOptions().getStatusListener() != null) {
+            scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+            long rateInMillis = context.getOptions().getStatusListener().getUpdateInterval().toMillis();
+            scheduledExecutorService.scheduleAtFixedRate(() -> context.getOptions().getStatusListener().provideStatus(context.getStats().copy()),
+                                                         rateInMillis,
+                                                         rateInMillis,
+                                                         TimeUnit.MILLISECONDS);
+        }
+
         final KeyMaster copyMaster = FileStoreFactory.buildCopyMaster(context, workQueue, executorService);
         KeyMaster deleteMaster = null;
 
@@ -51,7 +62,10 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
                     log.info("mirror: completed");
                     break;
                 }
-                if (Sleep.sleep(100)) return;
+                if (Sleep.sleep(100)) {
+                    context.getStats().completedFully.set(false);
+                    return context.getStats();
+                }
             }
 
         } catch (Exception e) {
@@ -62,9 +76,14 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
             if (deleteMaster != null) {
                 try { deleteMaster.stop(); } catch (Exception e) { log.error("Error stopping deleteMaster: "+e, e); }
             }
+            if (scheduledExecutorService != null) {
+                scheduledExecutorService.shutdownNow();
+            }
             // this will wait for currently executing tasks to finish, but there should be none by now
             executorService.shutdown();
         }
+
+        return context.getStats();
     }
 
     public static int getMaxQueueCapacity(MirrorOptions options) {