Fixed edge case where crawler may stop early and added more documentation to things.

src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java → src/main/java/net/jrtechs/www/utils/WrappedFileWriter.java

@@ -1,11 +1,23 @@
-package net.jrtechs.www.webCrawler;
+package net.jrtechs.www.utils;
 
 import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
 
-public class SteamdFileWriter
+
+/**
+ * Simple utility class to write the contents of a string to a
+ * file.
+ *
+ * @author Jeffery Russell 11-24-18
+ */
+public class WrappedFileWriter
 {
+    /**
+     * Writes the contents of a string to a file.
+     *
+     * @param data data to be included in the file
+     * @param fileName name of the file to write to.
+     */
     public static void writeToFile(String data, String fileName)
     {
         BufferedWriter writer;
@@ -13,7 +25,7 @@ public class SteamdFileWriter
         {
             File file = new File(fileName);
             file.createNewFile();
-            writer = new BufferedWriter(new FileWriter(file));
+            writer = new BufferedWriter(new java.io.FileWriter(file));
             writer.write(data);
             writer.flush();
             writer.close();

src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java

@@ -45,7 +45,6 @@ public class APIThrottler
         long currTime = getCurrentTimeInMS();
 
         return currTime > lastQuery + waitTime;
-
     }
 
 

src/main/java/net/jrtechs/www/webCrawler/FileIO.java

@@ -2,6 +2,7 @@ package net.jrtechs.www.webCrawler;
 
 import net.jrtechs.www.server.Player;
 import net.jrtechs.www.utils.FileReader;
+import net.jrtechs.www.utils.WrappedFileWriter;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
@@ -21,7 +22,7 @@ import java.util.List;
 public class FileIO
 {
     /** Base directory to store all the data */
-    private String baseFilaPath;
+    private String baseFilePath;
 
     /**
      * Initalizes the base directory
@@ -29,7 +30,7 @@ public class FileIO
      */
     public FileIO(String basePath)
     {
-        this.baseFilaPath = basePath;
+        this.baseFilePath = basePath;
     }
 
 
@@ -42,7 +43,7 @@ public class FileIO
      */
     private String getURL(String id)
     {
-        return baseFilaPath + id + ".json";
+        return baseFilePath + id + ".json";
     }
 
 
@@ -55,7 +56,7 @@ public class FileIO
      */
     public boolean playerExists(String id)
     {
-        String fileName = baseFilaPath + id + ".json";
+        String fileName = baseFilePath + id + ".json";
 
         return new File(fileName).isFile();
     }
@@ -119,8 +120,8 @@ public class FileIO
         object.put("date", getDate());
         object.put("friends", friendIDS);
 
-        String fileName = baseFilaPath + player.getId() + ".json";
+        String fileName = baseFilePath + player.getId() + ".json";
 
-        SteamdFileWriter.writeToFile(object.toString(4), fileName);
+        WrappedFileWriter.writeToFile(object.toString(4), fileName);
     }
 }

src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java

@@ -64,7 +64,9 @@ public class SteamWebCrawler
      */
     private void shiftNamelessToDownload()
     {
-        if(this.downlaodQueue.isEmpty() && !this.namelessQueue.isEmpty())
+        //this is a while instead of if because the getfull players query fails
+        //once in a blue moon
+        while(this.downlaodQueue.isEmpty() && !this.namelessQueue.isEmpty())
         {
             List<String> winners = new ArrayList<>();
             for(int i = 0; i < (100 < namelessQueue.size()? 100: namelessQueue.size()); i++)