Skip to content

Commit

Permalink
Merge pull request #631 from internetarchive/update-deps
Browse files Browse the repository at this point in the history 
Update dependencies for 3.6.0 release
  • Loading branch information
@ato
ato authored Nov 29, 2024
2 parents 1195bed + 6b10473 commit 3fbf1bc
Show file tree
Hide file tree
Showing 11 changed files with 43 additions and 71 deletions.
24 changes: 9 additions & 15 deletions commons/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,28 +42,22 @@
<version>2.6</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.0.4</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.9.0</version>
<version>3.11.1</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.14.0</version>
<version>2.18.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.6</version>
<version>1.17.1</version>
<scope>compile</scope>
</dependency>
<dependency>
Expand All @@ -75,7 +69,7 @@
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.1</version>
<version>1.9.0</version>
<scope>compile</scope>
</dependency>
<dependency>
Expand All @@ -87,7 +81,7 @@
<dependency>
<groupId>dnsjava</groupId>
<artifactId>dnsjava</artifactId>
<version>3.6.0</version>
<version>3.6.2</version>
<scope>compile</scope>
</dependency>
<!--
Expand Down Expand Up @@ -145,19 +139,19 @@
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>5.6.0</version>
<version>5.6.2</version>
</dependency>

<dependency>
<groupId>org.netpreserve.commons</groupId>
<artifactId>webarchive-commons</artifactId>
<version>1.1.10</version>
<version>1.2.0</version>
</dependency>

<dependency>
<groupId>com.jcraft</groupId>
<groupId>com.github.mwiede</groupId>
<artifactId>jsch</artifactId>
<version>0.1.54</version>
<version>0.2.21</version>
</dependency>
</dependencies>
<build>
Expand Down
23 changes: 4 additions & 19 deletions contrib/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,18 @@
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>5.18.0</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>3.0.1</version>
<version>5.23.0</version>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<version>3.1</version>
<version>5.5.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>3.8.0</version>
<version>3.9.0</version>
</dependency>
<dependency>
<groupId>com.rethinkdb</groupId>
Expand All @@ -53,17 +48,7 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.9</version>
</dependency>
<dependency>
<groupId>org.java-websocket</groupId>
<artifactId>Java-WebSocket</artifactId>
<version>1.5.2</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.16.1</version>
<version>2.11.0</version>
</dependency>
</dependencies>
<build>
Expand Down
5 changes: 1 addition & 4 deletions contrib/src/main/java/org/archive/modules/recrawl/FetchHistoryHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import java.util.HashMap;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.archive.modules.CrawlURI;

/**
Expand All @@ -32,8 +30,7 @@
*
*/
public class FetchHistoryHelper {
private static final Log logger = LogFactory.getLog(FetchHistoryHelper.class);
/**
/**
* key for storing timestamp in crawl history map.
*/
public static final String A_TIMESTAMP = ".ts";
Expand Down
16 changes: 8 additions & 8 deletions contrib/src/main/java/org/archive/modules/recrawl/wbm/WbmPersistLoadProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpRequest;
Expand All @@ -57,6 +55,8 @@
import org.archive.util.ArchiveUtils;
import org.archive.util.DateUtils;

import static java.lang.System.Logger.Level.ERROR;

/**
* A {@link Processor} for retrieving recrawl info from remote Wayback Machine index.
* This is currently in the early stage of experiment. Both low-level protocol and WBM API
Expand All @@ -69,7 +69,7 @@
* @author Kenji Nagahashi.
*/
public class WbmPersistLoadProcessor extends Processor {
private static final Log log = LogFactory.getLog(WbmPersistLoadProcessor.class);
private static final System.Logger log = System.getLogger(WbmPersistLoadProcessor.class.getName());

private HttpClient client;
private PoolingHttpClientConnectionManager conman;
Expand Down Expand Up @@ -415,17 +415,17 @@ protected InputStream getCDX(String qurl) throws InterruptedException, IOExcepti
cumulativeFetchTime.addAndGet(System.currentTimeMillis() - t0);
StatusLine sl = resp.getStatusLine();
if (sl.getStatusCode() != 200) {
log.error("GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase());
log.log(ERROR, "GET " + url + " failed with status=" + sl.getStatusCode() + " " + sl.getReasonPhrase());
entity = resp.getEntity();
entity.getContent().close();
entity = null;
continue;
}
entity = resp.getEntity();
} catch (IOException ex) {
log.error("GEt " + url + " failed with error " + ex.getMessage());
log.log(ERROR, "GEt " + url + " failed with error " + ex.getMessage());
} catch (Exception ex) {
log.error("GET " + url + " failed with error ", ex);
log.log(ERROR, "GET " + url + " failed with error ", ex);
}
} while (entity == null && ++attempts < 3);
if (entity == null) {
Expand All @@ -440,15 +440,15 @@ protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedExce
try {
is = getCDX(curi.toString());
} catch (IOException ex) {
log.error(ex.getMessage());
log.log(ERROR, ex.getMessage());
errorCount.incrementAndGet();
return ProcessResult.PROCEED;
}
Map<String, Object> info = null;
try {
info = getLastCrawl(is);
} catch (IOException ex) {
log.error("error parsing response", ex);
log.log(ERROR, "error parsing response", ex);
} finally {
if (is != null)
ArchiveUtils.closeQuietly(is);
Expand Down
8 changes: 4 additions & 4 deletions engine/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,14 @@
</dependency>
<!-- jaxb is no longer included in jdk11+ -->
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<version>4.0.2</version>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>2.3.1</version>
<version>4.0.5</version>
<scope>runtime</scope>
</dependency>
<dependency>
Expand Down
6 changes: 3 additions & 3 deletions engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
import java.util.HashMap;
import java.util.Map;

import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
import jakarta.xml.bind.annotation.XmlRootElement;
import jakarta.xml.bind.annotation.XmlTransient;
import jakarta.xml.bind.annotation.XmlType;

import org.apache.commons.lang.StringUtils;
import org.restlet.ext.xml.XmlWriter;
Expand Down
4 changes: 2 additions & 2 deletions engine/src/main/java/org/archive/crawler/restlet/models/ScriptModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
import java.util.List;
import java.util.Map;

import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
import jakarta.xml.bind.annotation.XmlRootElement;
import jakarta.xml.bind.annotation.XmlType;

import org.archive.crawler.restlet.ScriptingConsole;

Expand Down
4 changes: 2 additions & 2 deletions engine/src/test/java/org/archive/crawler/restlet/XmlMarshallerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import java.util.LinkedHashMap;
import java.util.Map;

import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
import jakarta.xml.bind.annotation.XmlRootElement;
import jakarta.xml.bind.annotation.XmlType;

import junit.framework.TestCase;

Expand Down
13 changes: 4 additions & 9 deletions modules/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,23 +82,18 @@
<dependency>
<groupId>com.github.crawler-commons</groupId>
<artifactId>crawler-commons</artifactId>
<version>1.0</version>
<version>1.4</version>
</dependency>
<dependency>
<groupId>com.github.bbottema</groupId>
<artifactId>java-socks-proxy-server</artifactId>
<version>1.1.0</version>
<version>4.1.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.54</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>3.0.1</version>
<version>3.0.3</version>
</dependency>
</dependencies>
<build>
Expand All @@ -120,6 +115,6 @@
</build>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<groovy.version>4.0.21</groovy.version>
<groovy.version>4.0.24</groovy.version>
</properties>
</project>
5 changes: 3 additions & 2 deletions modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -984,15 +984,15 @@ public static FetchHTTP newTestFetchHttp(String userAgentString) {

protected FetchHTTP makeSocksModule() throws Exception {
// configure our test server
socksServer = new SocksServer();
socksServer = new SocksServer(7800);


// SocksServers opens the socket on a background thread, so we supply a socket factory
// wrapper to let us block until the socket is opened and to report any exception that
// occurs when binding. This works around a race where we connect to the server before
// it is listening.
CompletableFuture<Void> socksServerStartedFuture = new CompletableFuture<>();
socksServer.start(7800, new ServerSocketFactory() {
socksServer.setFactory(new ServerSocketFactory() {
ServerSocketFactory defaultSocketFactory = ServerSocketFactory.getDefault();

@Override
Expand All @@ -1017,6 +1017,7 @@ public ServerSocket createServerSocket(int port, int backlog, InetAddress ifAddr
}
}
});
socksServer.start();
socksServerStartedFuture.get(30, TimeUnit.SECONDS);

FetchHTTP fetchHttp = newSocksTestFetchHttp(getUserAgentString(), "localhost", 7800);
Expand Down
6 changes: 3 additions & 3 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,17 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
<version>4.5.14</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.5.13</version>
<version>4.5.14</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.1</version>
<version>4.13.2</version>
</dependency>
</dependencies>
</dependencyManagement>
Expand Down

0 comments on commit 3fbf1bc

Please sign in to comment.