Updated the crawler to be more smart about error codes and not crash on a http error 500 or 401.

pom.xml

@@ -16,6 +16,9 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
+
+
+    <gson.version>2.8.5</gson.version>
   </properties>
 
   <dependencies>
@@ -61,6 +64,13 @@
       <version>1.3.0</version>
     </dependency>
 
+    <!-- JSON utils -->
+    <dependency>
+      <groupId>com.google.code.gson</groupId>
+      <artifactId>gson</artifactId>
+      <version>${gson.version}</version>
+    </dependency>
+
     <!-- https://mvnrepository.com/artifact/org.janusgraph/janusgraph-core -->
     <!--<dependency>-->
       <!--<groupId>org.janusgraph</groupId>-->

runCrawler.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# run - this script enters a infinite loop to ensure that
+# the socket server does not go down
+#
+# 7/14/18 Jeffery Russell
+
+while true
+do java -cp SteamFriendsGraph-0.1-jar-with-dependencies.jar net.jrtechs.www.webCrawler.SteamWebCrawler
+done

src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java

@@ -4,6 +4,7 @@ import net.jrtechs.www.server.Player;
 import net.jrtechs.www.utils.ConfigLoader;
 
 import net.jrtechs.www.utils.WebScraper;
+import net.jrtechs.www.webCrawler.APIThrottler;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
@@ -58,9 +59,43 @@ public class APIConnection
 
         try
         {
-            new JSONObject(WebScraper
-                    .getWebsite(this.baseURL + this.friendListURL +
-                            this.apiKey + "&steamid=" + steamid))
+            String apiData = "";
+            try
+            {
+                apiData = WebScraper
+                        .getWebsite(this.baseURL + this.friendListURL +
+                                this.apiKey + "&steamid=" + steamid);
+            }
+            catch (SteamConnectionException e)
+            {
+                switch (e.getError())
+                {
+                    case RESTRICTED:
+                    {
+                        //This is fine
+                        System.out.println("Private profile: " + steamid);
+                        return friendsId;
+                    }
+                    case CONNECTION:
+                    {
+                        //spooky 500 error :(
+                        new APIThrottler().wait(120);
+
+                        try
+                        {
+                            apiData = WebScraper
+                                    .getWebsite(this.baseURL + this.friendListURL +
+                                            this.apiKey + "&steamid=" + steamid);
+                        }
+                        catch (SteamConnectionException exception2)
+                        {
+                            throw new Exception("Everything is dead");
+                        }
+                    }
+                }
+            }
+
+            new JSONObject(apiData)
                     .getJSONObject("friendslist")
                     .getJSONArray("friends").toList()
                     .forEach(f->
@@ -69,8 +104,8 @@ public class APIConnection
         }
         catch (Exception ex)
         {
-            System.out.println("Friends not public :(");
-            //ex.printStackTrace();
+            ex.printStackTrace();
+            System.exit(-1);
         }
 
         return friendsId;
@@ -103,8 +138,17 @@ public class APIConnection
             }
 
             System.out.println(queryUrl);
-            JSONArray names = new JSONObject(WebScraper.getWebsite(queryUrl))
-                    .getJSONObject("response").getJSONArray("players");
+            JSONArray names;
+            try
+            {
+                names = new JSONObject(WebScraper.getWebsite(queryUrl))
+                        .getJSONObject("response").getJSONArray("players");
+            }
+            catch (SteamConnectionException ex)
+            {
+                //meh
+                return map;
+            }
 
             for(int i = 0; i < names.length(); i++)
             {
@@ -143,6 +187,7 @@ public class APIConnection
     }
 
 
+
     /**
      * Returns the name of the player with a specific steam id
      *
@@ -151,9 +196,18 @@ public class APIConnection
      */
     public String getPlayerName(String steamid)
     {
-        JSONObject response = new JSONObject(WebScraper
-                .getWebsite(this.baseURL + this.playerInfoURL +
-                        this.apiKey + "&steamids=" + steamid));
+        JSONObject response;
+        try
+        {
+            response = new JSONObject(WebScraper
+                    .getWebsite(this.baseURL + this.playerInfoURL +
+                            this.apiKey + "&steamids=" + steamid));
+        }
+        catch (SteamConnectionException ex)
+        {
+            return "";
+        }
+
 
         if(response.has("response"))
         {

src/main/java/net/jrtechs/www/SteamAPI/ConnectionErrors.java

@@ -0,0 +1,11 @@
+package net.jrtechs.www.SteamAPI;
+
+/**
+ * @author Jeffery Russell 11-19-18
+ */
+public enum ConnectionErrors
+{
+    VALID,
+    CONNECTION,
+    RESTRICTED
+}

src/main/java/net/jrtechs/www/SteamAPI/SteamConnectionException.java

@@ -0,0 +1,20 @@
+package net.jrtechs.www.SteamAPI;
+
+
+/**
+ * @author Jeffery Russell 11-19-18
+ */
+public class SteamConnectionException extends Exception
+{
+    private ConnectionErrors error;
+
+    public SteamConnectionException(ConnectionErrors error)
+    {
+        this.error = error;
+    }
+
+
+    public ConnectionErrors getError() {
+        return error;
+    }
+}

src/main/java/net/jrtechs/www/utils/WebScraper.java

@@ -1,5 +1,8 @@
 package net.jrtechs.www.utils;
 
+import net.jrtechs.www.SteamAPI.ConnectionErrors;
+import net.jrtechs.www.SteamAPI.SteamConnectionException;
+
 import java.io.*;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -17,22 +20,39 @@ public class WebScraper
      * @param link to open
      * @return source code of website as a single string
      */
-    public static String getWebsite(String link)
+    public static String getWebsite(String link) throws SteamConnectionException
     {
         try
         {
             URL url = new URL(link);
+            HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+            connection.setRequestMethod("GET");
+            connection.connect();
+
+            int code = connection.getResponseCode();
+
+            switch (code)
+            {
+                case 401:
+                    throw new SteamConnectionException(ConnectionErrors.RESTRICTED);
+                case 500:
+                    throw new SteamConnectionException(ConnectionErrors.CONNECTION);
+                default:
+            }
 
             BufferedReader br = new BufferedReader(
                     new InputStreamReader(url.openStream())
             );
             return WebScraper.getBufferedReaderData(br);
         }
+        catch (SteamConnectionException e)
+        {
+            throw e;
+        }
         catch (Exception e)
         {
-            e.printStackTrace();
+            throw new SteamConnectionException(ConnectionErrors.CONNECTION);
         }
-        return "";
     }