diff --git a/pom.xml b/pom.xml
index f3d5e3b..68de8f9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,6 +16,9 @@
UTF-8
1.8
1.8
+
+
+ 2.8.5
@@ -61,6 +64,13 @@
1.3.0
+
+
+ com.google.code.gson
+ gson
+ ${gson.version}
+
+
diff --git a/runCrawler.sh b/runCrawler.sh
new file mode 100644
index 0000000..cfa952b
--- /dev/null
+++ b/runCrawler.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# run - this script enters a infinite loop to ensure that
+# the socket server does not go down
+#
+# 7/14/18 Jeffery Russell
+
+while true
+do java -cp SteamFriendsGraph-0.1-jar-with-dependencies.jar net.jrtechs.www.webCrawler.SteamWebCrawler
+done
\ No newline at end of file
diff --git a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java
index c3ea12b..e60de73 100644
--- a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java
+++ b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java
@@ -4,6 +4,7 @@ import net.jrtechs.www.server.Player;
import net.jrtechs.www.utils.ConfigLoader;
import net.jrtechs.www.utils.WebScraper;
+import net.jrtechs.www.webCrawler.APIThrottler;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -58,9 +59,43 @@ public class APIConnection
try
{
- new JSONObject(WebScraper
- .getWebsite(this.baseURL + this.friendListURL +
- this.apiKey + "&steamid=" + steamid))
+ String apiData = "";
+ try
+ {
+ apiData = WebScraper
+ .getWebsite(this.baseURL + this.friendListURL +
+ this.apiKey + "&steamid=" + steamid);
+ }
+ catch (SteamConnectionException e)
+ {
+ switch (e.getError())
+ {
+ case RESTRICTED:
+ {
+ //This is fine
+ System.out.println("Private profile: " + steamid);
+ return friendsId;
+ }
+ case CONNECTION:
+ {
+ //spooky 500 error :(
+ new APIThrottler().wait(120);
+
+ try
+ {
+ apiData = WebScraper
+ .getWebsite(this.baseURL + this.friendListURL +
+ this.apiKey + "&steamid=" + steamid);
+ }
+ catch (SteamConnectionException exception2)
+ {
+ throw new Exception("Everything is dead");
+ }
+ }
+ }
+ }
+
+ new JSONObject(apiData)
.getJSONObject("friendslist")
.getJSONArray("friends").toList()
.forEach(f->
@@ -69,8 +104,8 @@ public class APIConnection
}
catch (Exception ex)
{
- System.out.println("Friends not public :(");
- //ex.printStackTrace();
+ ex.printStackTrace();
+ System.exit(-1);
}
return friendsId;
@@ -103,8 +138,17 @@ public class APIConnection
}
System.out.println(queryUrl);
- JSONArray names = new JSONObject(WebScraper.getWebsite(queryUrl))
- .getJSONObject("response").getJSONArray("players");
+ JSONArray names;
+ try
+ {
+ names = new JSONObject(WebScraper.getWebsite(queryUrl))
+ .getJSONObject("response").getJSONArray("players");
+ }
+ catch (SteamConnectionException ex)
+ {
+ //meh
+ return map;
+ }
for(int i = 0; i < names.length(); i++)
{
@@ -143,6 +187,7 @@ public class APIConnection
}
+
/**
* Returns the name of the player with a specific steam id
*
@@ -151,9 +196,18 @@ public class APIConnection
*/
public String getPlayerName(String steamid)
{
- JSONObject response = new JSONObject(WebScraper
- .getWebsite(this.baseURL + this.playerInfoURL +
- this.apiKey + "&steamids=" + steamid));
+ JSONObject response;
+ try
+ {
+ response = new JSONObject(WebScraper
+ .getWebsite(this.baseURL + this.playerInfoURL +
+ this.apiKey + "&steamids=" + steamid));
+ }
+ catch (SteamConnectionException ex)
+ {
+ return "";
+ }
+
if(response.has("response"))
{
diff --git a/src/main/java/net/jrtechs/www/SteamAPI/ConnectionErrors.java b/src/main/java/net/jrtechs/www/SteamAPI/ConnectionErrors.java
new file mode 100644
index 0000000..10fd16d
--- /dev/null
+++ b/src/main/java/net/jrtechs/www/SteamAPI/ConnectionErrors.java
@@ -0,0 +1,11 @@
+package net.jrtechs.www.SteamAPI;
+
+/**
+ * @author Jeffery Russell 11-19-18
+ */
+public enum ConnectionErrors
+{
+ VALID,
+ CONNECTION,
+ RESTRICTED
+}
diff --git a/src/main/java/net/jrtechs/www/SteamAPI/SteamConnectionException.java b/src/main/java/net/jrtechs/www/SteamAPI/SteamConnectionException.java
new file mode 100644
index 0000000..3def6db
--- /dev/null
+++ b/src/main/java/net/jrtechs/www/SteamAPI/SteamConnectionException.java
@@ -0,0 +1,20 @@
+package net.jrtechs.www.SteamAPI;
+
+
+/**
+ * @author Jeffery Russell 11-19-18
+ */
+public class SteamConnectionException extends Exception
+{
+ private ConnectionErrors error;
+
+ public SteamConnectionException(ConnectionErrors error)
+ {
+ this.error = error;
+ }
+
+
+ public ConnectionErrors getError() {
+ return error;
+ }
+}
diff --git a/src/main/java/net/jrtechs/www/utils/WebScraper.java b/src/main/java/net/jrtechs/www/utils/WebScraper.java
index 2d955f9..7a74a5d 100644
--- a/src/main/java/net/jrtechs/www/utils/WebScraper.java
+++ b/src/main/java/net/jrtechs/www/utils/WebScraper.java
@@ -1,5 +1,8 @@
package net.jrtechs.www.utils;
+import net.jrtechs.www.SteamAPI.ConnectionErrors;
+import net.jrtechs.www.SteamAPI.SteamConnectionException;
+
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -17,22 +20,39 @@ public class WebScraper
* @param link to open
* @return source code of website as a single string
*/
- public static String getWebsite(String link)
+ public static String getWebsite(String link) throws SteamConnectionException
{
try
{
URL url = new URL(link);
+ HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+
+ int code = connection.getResponseCode();
+
+ switch (code)
+ {
+ case 401:
+ throw new SteamConnectionException(ConnectionErrors.RESTRICTED);
+ case 500:
+ throw new SteamConnectionException(ConnectionErrors.CONNECTION);
+ default:
+ }
BufferedReader br = new BufferedReader(
new InputStreamReader(url.openStream())
);
return WebScraper.getBufferedReaderData(br);
}
+ catch (SteamConnectionException e)
+ {
+ throw e;
+ }
catch (Exception e)
{
- e.printStackTrace();
+ throw new SteamConnectionException(ConnectionErrors.CONNECTION);
}
- return "";
}