# v1.2.0

- Change project to maven format
- Added apache commons httpclient (fix some problems with urlconnection requests)

.project

@@ -10,8 +10,14 @@
 			<arguments>
 			</arguments>
 		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
 	</buildSpec>
 	<natures>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
 		<nature>org.eclipse.jdt.core.javanature</nature>
 	</natures>
 </projectDescription>

changelog

@@ -1,3 +1,7 @@
+# v1.2.0
+- Change project to maven format
+- Added apache commons httpclient (fix some problems with urlconnection requests)
+
 # v1.1.0
 - Added new attributes to tweet model: mentions, hashtags, id, permalink.
 

pom.xml

@@ -0,0 +1,26 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>me.jhenrique</groupId>
+	<artifactId>getoldtweets</artifactId>
+	<version>1.1.0</version>
+	<name>GetOldTweets</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.jsoup</groupId>
+			<artifactId>jsoup</artifactId>
+			<version>1.8.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.json</groupId>
+			<artifactId>json</artifactId>
+			<version>20151123</version>
+		</dependency>
+	 	<dependency>
+			<groupId>org.apache.httpcomponents</groupId>
+			<artifactId>httpclient</artifactId>
+			<version>4.5</version>
+		</dependency>
+	 </dependencies>
+
+</project>

src/me/jhenrique/main/Exporter.java → src/main/java/me/jhenrique/main/Exporter.java

@@ -21,7 +21,7 @@ public static void main(String[] args) {
 
 		if (args.length == 1 && args[0].equals("-h")) {
 			System.out.println("\nTo use this jar, you can pass the folowing attributes:");
-			System.out.println("   username: Username of a specific twitter account (whitout @)");
+			System.out.println("   username: Username of a specific twitter account (without @)");
 			System.out.println("      since: The lower bound date (yyyy-mm-aa)");
 			System.out.println("      until: The upper bound date (yyyy-mm-aa)");
 			System.out.println("querysearch: A query text to be matched");

src/me/jhenrique/main/Main.java → src/main/java/me/jhenrique/main/Main.java

@@ -1,7 +1,5 @@
 package me.jhenrique.main;
 
-import java.util.List;
-
 import me.jhenrique.manager.TweetManager;
 import me.jhenrique.manager.TwitterCriteria;
 import me.jhenrique.model.Tweet;
@@ -37,11 +35,10 @@ public static void main(String[] args) {
 		 **/
 		criteria = TwitterCriteria.create()
 				.setQuerySearch("europe refugees")
-				.setSince("2015-03-01")
+				.setSince("2015-05-01")
 				.setUntil("2015-09-30")
-				.setMaxTweets(120);
-		List<Tweet> allTweets = TweetManager.getTweets(criteria);
-		t = allTweets.get(0);
+				.setMaxTweets(1);
+		t = TweetManager.getTweets(criteria).get(0);
 
 		System.out.println("### Example 2 - Get tweets by query search [europe refugees]");
 		System.out.println("Username: " + t.getUsername());

src/me/jhenrique/manager/TweetManager.java → src/main/java/me/jhenrique/manager/TweetManager.java

@@ -1,18 +1,21 @@
 package me.jhenrique.manager;
 
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import me.jhenrique.model.Tweet;
 
+import org.apache.http.HttpEntity;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
 import org.json.JSONObject;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -25,6 +28,12 @@
  * @author Jefferson Henrique
  */
 public class TweetManager {
+
+	private static final HttpClient defaultHttpClient = HttpClients.createDefault();
+
+	static {
+		Logger.getLogger("org.apache.http").setLevel(Level.OFF);
+	}
 
 	/**
 	 * @param username A specific username (without @)
@@ -51,23 +60,10 @@ private static String getURLResponse(String username, String since, String until
 
 		String url = String.format("https://twitter.com/i/search/timeline?f=realtime&q=%s&src=typd&max_position=%s", URLEncoder.encode(appendQuery, "UTF-8"), scrollCursor);
 
-		URL obj = new URL(url);
-		HttpURLConnection con = (HttpURLConnection) obj.openConnection();
-
-		con.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36");
-		con.setRequestMethod("GET");
-
-		BufferedReader in = new BufferedReader(
-		        new InputStreamReader(con.getInputStream()));
-		String inputLine;
-		StringBuffer response = new StringBuffer();
-
-		while ((inputLine = in.readLine()) != null) {
-			response.append(inputLine);
-		}
-		in.close();
+		HttpGet httpGet = new HttpGet(url);
+		HttpEntity resp = defaultHttpClient.execute(httpGet).getEntity();
 
-		return response.toString();
+		return EntityUtils.toString(resp);
 	}
 
 	/**
@@ -115,8 +111,8 @@ public static List<Tweet> getTweets(TwitterCriteria criteria) {
 					t.setDate(date);
 					t.setRetweets(retweets);
 					t.setFavorites(favorites);
-					t.setMentions(processMentions(txt));
-					t.setHashtags(processHashtags(txt));
+					t.setMentions(processTerms("(@\\w*)", txt));
+					t.setHashtags(processTerms("(#\\w*)", txt));
 					t.setGeo(geo);
 
 					results.add(t);
@@ -133,20 +129,9 @@ public static List<Tweet> getTweets(TwitterCriteria criteria) {
 		return results;
 	}
 
-	private static String processMentions(String tweetText) {
-		StringBuilder sb = new StringBuilder();
-		Matcher matcher = Pattern.compile("(@\\w*)").matcher(tweetText);
-		while (matcher.find()) {
-			sb.append(matcher.group());
-			sb.append(" ");
-		}
-
-		return sb.toString().trim();
-	}
-
-	private static String processHashtags(String tweetText) {
+	private static String processTerms(String patternS, String tweetText) {
 		StringBuilder sb = new StringBuilder();
-		Matcher matcher = Pattern.compile("(#\\w*)").matcher(tweetText);
+		Matcher matcher = Pattern.compile(patternS).matcher(tweetText);
 		while (matcher.find()) {
 			sb.append(matcher.group());
 			sb.append(" ");