diff --git a/src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java b/src/main/java/net/jrtechs/www/utils/WrappedFileWriter.java similarity index 53% rename from src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java rename to src/main/java/net/jrtechs/www/utils/WrappedFileWriter.java index 318fa3b..7dfeb3c 100644 --- a/src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java +++ b/src/main/java/net/jrtechs/www/utils/WrappedFileWriter.java @@ -1,11 +1,23 @@ -package net.jrtechs.www.webCrawler; +package net.jrtechs.www.utils; import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; -public class SteamdFileWriter + +/** + * Simple utility class to write the contents of a string to a + * file. + * + * @author Jeffery Russell 11-24-18 + */ +public class WrappedFileWriter { + /** + * Writes the contents of a string to a file. + * + * @param data data to be included in the file + * @param fileName name of the file to write to. + */ public static void writeToFile(String data, String fileName) { BufferedWriter writer; @@ -13,7 +25,7 @@ public class SteamdFileWriter { File file = new File(fileName); file.createNewFile(); - writer = new BufferedWriter(new FileWriter(file)); + writer = new BufferedWriter(new java.io.FileWriter(file)); writer.write(data); writer.flush(); writer.close(); diff --git a/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java b/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java index 578fa2a..01ba56a 100644 --- a/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java +++ b/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java @@ -45,7 +45,6 @@ public class APIThrottler long currTime = getCurrentTimeInMS(); return currTime > lastQuery + waitTime; - } diff --git a/src/main/java/net/jrtechs/www/webCrawler/FileIO.java b/src/main/java/net/jrtechs/www/webCrawler/FileIO.java index 61fa5eb..ca6b444 100644 --- a/src/main/java/net/jrtechs/www/webCrawler/FileIO.java +++ b/src/main/java/net/jrtechs/www/webCrawler/FileIO.java @@ -2,6 +2,7 @@ package net.jrtechs.www.webCrawler; import net.jrtechs.www.server.Player; import net.jrtechs.www.utils.FileReader; +import net.jrtechs.www.utils.WrappedFileWriter; import org.json.JSONArray; import org.json.JSONObject; @@ -21,7 +22,7 @@ import java.util.List; public class FileIO { /** Base directory to store all the data */ - private String baseFilaPath; + private String baseFilePath; /** * Initalizes the base directory @@ -29,7 +30,7 @@ public class FileIO */ public FileIO(String basePath) { - this.baseFilaPath = basePath; + this.baseFilePath = basePath; } @@ -42,7 +43,7 @@ public class FileIO */ private String getURL(String id) { - return baseFilaPath + id + ".json"; + return baseFilePath + id + ".json"; } @@ -55,7 +56,7 @@ public class FileIO */ public boolean playerExists(String id) { - String fileName = baseFilaPath + id + ".json"; + String fileName = baseFilePath + id + ".json"; return new File(fileName).isFile(); } @@ -119,8 +120,8 @@ public class FileIO object.put("date", getDate()); object.put("friends", friendIDS); - String fileName = baseFilaPath + player.getId() + ".json"; + String fileName = baseFilePath + player.getId() + ".json"; - SteamdFileWriter.writeToFile(object.toString(4), fileName); + WrappedFileWriter.writeToFile(object.toString(4), fileName); } } diff --git a/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java b/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java index d74bfc6..b708aec 100644 --- a/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java +++ b/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java @@ -64,7 +64,9 @@ public class SteamWebCrawler */ private void shiftNamelessToDownload() { - if(this.downlaodQueue.isEmpty() && !this.namelessQueue.isEmpty()) + //this is a while instead of if because the getfull players query fails + //once in a blue moon + while(this.downlaodQueue.isEmpty() && !this.namelessQueue.isEmpty()) { List winners = new ArrayList<>(); for(int i = 0; i < (100 < namelessQueue.size()? 100: namelessQueue.size()); i++)