Skip to content

Commit d64fad6

Browse files
wyu4wyu4
committed
Fix HTTP 403/404 errors
The repository of Sky Clock had been privated, and the Wiki for sky implemented some sort of anti-bot protection.
1 parent 598ac00 commit d64fad6

5 files changed

Lines changed: 34 additions & 50 deletions

File tree

‎dependency-reduced-pom.xml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,12 @@
7171
<windowTitle>SkyEvents</windowTitle>
7272
</singleInstance>
7373
<versionInfo>
74-
<fileVersion>1.0.0.0</fileVersion>
75-
<txtFileVersion>1.0.0.0</txtFileVersion>
74+
<fileVersion>1.0.0.1</fileVersion>
75+
<txtFileVersion>1.0.0.1</txtFileVersion>
7676
<fileDescription>Track Sky events!</fileDescription>
7777
<copyright>None</copyright>
7878
<companyName>Wallibe</companyName>
79-
<productVersion>0.0.0.0</productVersion>
79+
<productVersion>1.0.0.1</productVersion>
8080
<txtProductVersion>${project.version}</txtProductVersion>
8181
<productName>Sky Events</productName>
8282
<internalName>WingWatch</internalName>
@@ -89,8 +89,8 @@
8989
</plugins>
9090
</build>
9191
<properties>
92+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
9293
<maven.compiler.target>21</maven.compiler.target>
9394
<maven.compiler.source>21</maven.compiler.source>
94-
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
9595
</properties>
9696
</project>

‎pom.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,12 +78,12 @@
7878
<windowTitle>SkyEvents</windowTitle>
7979
</singleInstance>
8080
<versionInfo>
81-
<fileVersion>1.0.0.0</fileVersion>
82-
<txtFileVersion>1.0.0.0</txtFileVersion>
81+
<fileVersion>0.1.0.1</fileVersion>
82+
<txtFileVersion>0.1.0.1</txtFileVersion>
8383
<fileDescription>Track Sky events!</fileDescription>
8484
<copyright>None</copyright>
8585
<companyName>Wallibe</companyName>
86-
<productVersion>0.0.0.0</productVersion>
86+
<productVersion>0.1.0.1</productVersion>
8787
<txtProductVersion>${project.version}</txtProductVersion>
8888
<productName>Sky Events</productName>
8989
<internalName>WingWatch</internalName>

‎src/main/java/com/WingWatch/WebScraping/Requests.java‎

Lines changed: 14 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,24 @@
11
package com.WingWatch.WebScraping;
22

3-
import java.io.BufferedReader;
43
import java.io.IOException;
5-
import java.io.InputStream;
6-
import java.io.InputStreamReader;
7-
import java.net.HttpURLConnection;
8-
import java.net.URI;
94
import java.net.URISyntaxException;
10-
import java.net.URL;
115

12-
public abstract class Requests {
13-
private static InputStream getInputStream(String targetUrl) throws IOException, URISyntaxException {
14-
URL url;
15-
HttpURLConnection connection;
16-
17-
// Create a URL Object
18-
url = new URI(targetUrl).toURL();
19-
20-
connection = (HttpURLConnection) url.openConnection();
21-
connection.setRequestMethod("GET"); // Set the method to s GET Request
6+
import org.jsoup.Jsoup;
7+
import org.jsoup.nodes.Document;
228

23-
return connection.getInputStream();
24-
}
9+
public abstract class Requests {
10+
public static String requestGetString(String targetUrl) throws IOException, URISyntaxException {
11+
Document document = Jsoup.connect(targetUrl)
12+
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/121.0.0.0 Safari/537.36")
13+
.header("Accept-Language", "en-US,en;q=0.9")
14+
.header("Accept-Encoding", "gzip, deflate")
15+
.header("Referer", "https://example.com")
16+
.timeout(10000) // replaces connect + read timeout
17+
.ignoreContentType(true) // allows JSON / plain text too
18+
.get();
2519

26-
public static StringBuilder requestGetString(String targetUrl) throws IOException, URISyntaxException {
27-
final StringBuilder result = new StringBuilder();
28-
InputStream stream = getInputStream(targetUrl);
20+
document.outputSettings().charset("UTF-8");
2921

30-
// Perform the request
31-
BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
32-
for (String line; (line = reader.readLine()) != null; ) {
33-
result.append(line);
34-
}
35-
stream.close();
36-
return result;
22+
return document.outerHtml();
3723
}
3824
}

‎src/main/java/com/WingWatch/WebScraping/SkyClockUtils.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ public abstract class SkyClockUtils {
1616

1717
public static final String US_PACIFIC_TIME_ZONE = "US_PACIFIC_TIME_ZONE";
1818

19-
private static volatile HashMap<String, String> REGION_DATA = new HashMap<>();
19+
private static final HashMap<String, String> REGION_DATA = new HashMap<>();
2020

2121
private static HashMap<String, String> getVariables(String url) throws IOException, URISyntaxException {
22-
StringBuilder data = Requests.requestGetString(url);
22+
String data = Requests.requestGetString(url);
2323

2424
Matcher matcher = CONST_VAR_PATTERN.matcher(data);
2525

@@ -35,12 +35,14 @@ private static HashMap<String, String> getVariables(String url) throws IOExcepti
3535
}
3636

3737
public static void refreshData() {
38-
try {
39-
REGION_DATA = getVariables(REGION_URL);
40-
LocalCache.setValue(US_PACIFIC_TIME_ZONE, REGION_DATA.get(US_PACIFIC_TIME_ZONE));
41-
} catch (IOException | URISyntaxException e) {
42-
System.err.println("Could not refresh SkyClock data: " + e);
43-
}
38+
// try {
39+
// REGION_DATA = getVariables(REGION_URL);
40+
// LocalCache.setValue(US_PACIFIC_TIME_ZONE, REGION_DATA.get(US_PACIFIC_TIME_ZONE));
41+
// } catch (IOException | URISyntaxException e) {
42+
// System.err.println("Could not refresh SkyClock data: " + e);
43+
// LocalCache.setValue(US_PACIFIC_TIME_ZONE, "America/Los_Angeles");
44+
// }
45+
LocalCache.setValue(US_PACIFIC_TIME_ZONE, "America/Los_Angeles");
4446
}
4547

4648
public static String formatTimeLeft(Long remainingTime) {

‎src/main/java/com/WingWatch/WebScraping/WikiUtils.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@
66
import org.jsoup.nodes.Element;
77
import org.jsoup.nodes.TextNode;
88

9-
import java.io.File;
109
import java.io.IOException;
11-
import java.io.PrintStream;
1210
import java.net.URISyntaxException;
13-
import java.net.UnknownHostException;
1411
import java.time.ZoneId;
1512
import java.time.ZonedDateTime;
1613
import java.util.HashMap;
@@ -40,7 +37,7 @@ public abstract class WikiUtils {
4037

4138
private static String getSource(String wikiUrl) {
4239
try {
43-
return Requests.requestGetString(wikiUrl).toString();
40+
return Requests.requestGetString(wikiUrl);
4441
} catch (IOException | URISyntaxException e) {
4542
System.err.println("Error getting source " + wikiUrl + ": " + e.getMessage());
4643
return SOURCES.get(wikiUrl);
@@ -60,8 +57,7 @@ private static ZonedDateTime[] getPeriod(ZoneId zone, String dateText) throws Nu
6057
Integer year1 = null;
6158
Integer year2 = null;
6259

63-
for (int i = 0; i < dateData.length; i++) {
64-
String item = dateData[i];
60+
for (String item : dateData) {
6561
try {
6662
int number = Integer.parseInt(item);
6763
if (number <= 31) {
@@ -77,7 +73,7 @@ private static ZonedDateTime[] getPeriod(ZoneId zone, String dateText) throws Nu
7773
year2 = number;
7874
}
7975
}
80-
} catch(NumberFormatException e) {
76+
} catch (NumberFormatException e) {
8177
if (MONTHS.containsKey(item)) {
8278
if (month1 == null) {
8379
month1 = MONTHS.get(item);

0 commit comments

Comments
 (0)