diff --git a/.project b/.project index 5400a2b..1427d1f 100644 --- a/.project +++ b/.project @@ -10,8 +10,14 @@ + + org.eclipse.m2e.core.maven2Builder + + + + org.eclipse.m2e.core.maven2Nature org.eclipse.jdt.core.javanature diff --git a/changelog b/changelog index 4356c29..769faef 100644 --- a/changelog +++ b/changelog @@ -1,3 +1,7 @@ +# v1.2.0 +- Change project to maven format +- Added apache commons httpclient (fix some problems with urlconnection requests) + # v1.1.0 - Added new attributes to tweet model: mentions, hashtags, id, permalink. diff --git a/got.jar b/got.jar index 7ceb7c0..33390f4 100644 Binary files a/got.jar and b/got.jar differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..472d69d --- /dev/null +++ b/pom.xml @@ -0,0 +1,26 @@ + + 4.0.0 + me.jhenrique + getoldtweets + 1.1.0 + GetOldTweets + + + + org.jsoup + jsoup + 1.8.1 + + + org.json + json + 20151123 + + + org.apache.httpcomponents + httpclient + 4.5 + + + + \ No newline at end of file diff --git a/src/me/jhenrique/main/Exporter.java b/src/main/java/me/jhenrique/main/Exporter.java similarity index 99% rename from src/me/jhenrique/main/Exporter.java rename to src/main/java/me/jhenrique/main/Exporter.java index 2836ed8..a6ea4d6 100644 --- a/src/me/jhenrique/main/Exporter.java +++ b/src/main/java/me/jhenrique/main/Exporter.java @@ -21,7 +21,7 @@ public static void main(String[] args) { if (args.length == 1 && args[0].equals("-h")) { System.out.println("\nTo use this jar, you can pass the folowing attributes:"); - System.out.println(" username: Username of a specific twitter account (whitout @)"); + System.out.println(" username: Username of a specific twitter account (without @)"); System.out.println(" since: The lower bound date (yyyy-mm-aa)"); System.out.println(" until: The upper bound date (yyyy-mm-aa)"); System.out.println("querysearch: A query text to be matched"); diff --git a/src/me/jhenrique/main/Main.java b/src/main/java/me/jhenrique/main/Main.java similarity index 92% rename from src/me/jhenrique/main/Main.java rename to src/main/java/me/jhenrique/main/Main.java index 2fa7292..62bb88b 100644 --- a/src/me/jhenrique/main/Main.java +++ b/src/main/java/me/jhenrique/main/Main.java @@ -1,7 +1,5 @@ package me.jhenrique.main; -import java.util.List; - import me.jhenrique.manager.TweetManager; import me.jhenrique.manager.TwitterCriteria; import me.jhenrique.model.Tweet; @@ -37,11 +35,10 @@ public static void main(String[] args) { **/ criteria = TwitterCriteria.create() .setQuerySearch("europe refugees") - .setSince("2015-03-01") + .setSince("2015-05-01") .setUntil("2015-09-30") - .setMaxTweets(120); - List allTweets = TweetManager.getTweets(criteria); - t = allTweets.get(0); + .setMaxTweets(1); + t = TweetManager.getTweets(criteria).get(0); System.out.println("### Example 2 - Get tweets by query search [europe refugees]"); System.out.println("Username: " + t.getUsername()); diff --git a/src/me/jhenrique/manager/TweetManager.java b/src/main/java/me/jhenrique/manager/TweetManager.java similarity index 76% rename from src/me/jhenrique/manager/TweetManager.java rename to src/main/java/me/jhenrique/manager/TweetManager.java index 3c86fa3..c7e29c6 100644 --- a/src/me/jhenrique/manager/TweetManager.java +++ b/src/main/java/me/jhenrique/manager/TweetManager.java @@ -1,18 +1,21 @@ package me.jhenrique.manager; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import me.jhenrique.model.Tweet; +import org.apache.http.HttpEntity; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -25,6 +28,12 @@ * @author Jefferson Henrique */ public class TweetManager { + + private static final HttpClient defaultHttpClient = HttpClients.createDefault(); + + static { + Logger.getLogger("org.apache.http").setLevel(Level.OFF); + } /** * @param username A specific username (without @) @@ -51,23 +60,10 @@ private static String getURLResponse(String username, String since, String until String url = String.format("https://twitter.com/i/search/timeline?f=realtime&q=%s&src=typd&max_position=%s", URLEncoder.encode(appendQuery, "UTF-8"), scrollCursor); - URL obj = new URL(url); - HttpURLConnection con = (HttpURLConnection) obj.openConnection(); - - con.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36"); - con.setRequestMethod("GET"); - - BufferedReader in = new BufferedReader( - new InputStreamReader(con.getInputStream())); - String inputLine; - StringBuffer response = new StringBuffer(); - - while ((inputLine = in.readLine()) != null) { - response.append(inputLine); - } - in.close(); + HttpGet httpGet = new HttpGet(url); + HttpEntity resp = defaultHttpClient.execute(httpGet).getEntity(); - return response.toString(); + return EntityUtils.toString(resp); } /** @@ -115,8 +111,8 @@ public static List getTweets(TwitterCriteria criteria) { t.setDate(date); t.setRetweets(retweets); t.setFavorites(favorites); - t.setMentions(processMentions(txt)); - t.setHashtags(processHashtags(txt)); + t.setMentions(processTerms("(@\\w*)", txt)); + t.setHashtags(processTerms("(#\\w*)", txt)); t.setGeo(geo); results.add(t); @@ -133,20 +129,9 @@ public static List getTweets(TwitterCriteria criteria) { return results; } - private static String processMentions(String tweetText) { - StringBuilder sb = new StringBuilder(); - Matcher matcher = Pattern.compile("(@\\w*)").matcher(tweetText); - while (matcher.find()) { - sb.append(matcher.group()); - sb.append(" "); - } - - return sb.toString().trim(); - } - - private static String processHashtags(String tweetText) { + private static String processTerms(String patternS, String tweetText) { StringBuilder sb = new StringBuilder(); - Matcher matcher = Pattern.compile("(#\\w*)").matcher(tweetText); + Matcher matcher = Pattern.compile(patternS).matcher(tweetText); while (matcher.find()) { sb.append(matcher.group()); sb.append(" "); diff --git a/src/me/jhenrique/manager/TwitterCriteria.java b/src/main/java/me/jhenrique/manager/TwitterCriteria.java similarity index 100% rename from src/me/jhenrique/manager/TwitterCriteria.java rename to src/main/java/me/jhenrique/manager/TwitterCriteria.java diff --git a/src/me/jhenrique/model/Tweet.java b/src/main/java/me/jhenrique/model/Tweet.java similarity index 100% rename from src/me/jhenrique/model/Tweet.java rename to src/main/java/me/jhenrique/model/Tweet.java