diff --git a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java index f566eac..c3ea12b 100644 --- a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java +++ b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java @@ -69,7 +69,8 @@ public class APIConnection } catch (Exception ex) { - ex.printStackTrace(); + System.out.println("Friends not public :("); + //ex.printStackTrace(); } return friendsId; @@ -108,8 +109,12 @@ public class APIConnection for(int i = 0; i < names.length(); i++) { JSONObject player = names.getJSONObject(i); - map.put(player.getString("steamid"), - player.getString("personaname")); + + if(player.has("steamid") && player.has("personaname")) + { + map.put(player.getString("steamid"), + player.getString("personaname")); + } } } return map; diff --git a/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java b/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java index 80d44be..578fa2a 100644 --- a/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java +++ b/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java @@ -55,6 +55,7 @@ public class APIThrottler */ public void wait(int numofQueries) { + System.out.println("Hold the door:" + numofQueries); int totalWaitTime = numofQueries * waitTimePerQuerie; while(!queryAvailable(totalWaitTime)) diff --git a/src/main/java/net/jrtechs/www/webCrawler/FileIO.java b/src/main/java/net/jrtechs/www/webCrawler/FileIO.java index 42dbc4d..500e9ba 100644 --- a/src/main/java/net/jrtechs/www/webCrawler/FileIO.java +++ b/src/main/java/net/jrtechs/www/webCrawler/FileIO.java @@ -1,10 +1,12 @@ package net.jrtechs.www.webCrawler; import net.jrtechs.www.server.Player; +import org.json.JSONArray; import org.json.JSONObject; import java.io.File; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -30,6 +32,12 @@ public class FileIO } + private String getURL(String id) + { + return baseFilaPath + id + ".json"; + } + + /** * Determines if we already have the player * on disk. @@ -60,6 +68,28 @@ public class FileIO } + public List readFriends(String id) + { + String fileContents = FileReader.readFile(this.getURL(id)); + + JSONObject player = new JSONObject(fileContents); + + if(player.has("friends")) + { + List list = new ArrayList<>(); + + JSONArray jsonArray = player.getJSONArray("friends"); + + for(int i = 0 ; i < jsonArray.length();i++) + { + list.add(jsonArray.getString(i)); + } + return list; + } + return new ArrayList<>(); + } + + /** * Writes the player to the file. * diff --git a/src/main/java/net/jrtechs/www/webCrawler/FileReader.java b/src/main/java/net/jrtechs/www/webCrawler/FileReader.java new file mode 100644 index 0000000..6acf1c1 --- /dev/null +++ b/src/main/java/net/jrtechs/www/webCrawler/FileReader.java @@ -0,0 +1,44 @@ +package net.jrtechs.www.webCrawler; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Simple utility class for reading a file in as a + * {@link List} of strings + * + * @author Jeffery Russell 11-19-18 + */ +public class FileReader +{ + /** + * Reads a file and return's its contents in a array list of strings + * + * @return contents of file as a list of Strings + */ + public static String readFile(String filePath) + { + String result = ""; + try + { + BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(filePath))); + String line; + + while ((line = br.readLine()) != null) + { + result = result.concat(line); + } + br.close(); + } + catch (IOException e) + { + System.out.println("ERROR: unable to read file " + filePath); + } + return result; + } +} \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java b/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java index e0133df..d74bfc6 100644 --- a/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java +++ b/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java @@ -3,17 +3,14 @@ package net.jrtechs.www.webCrawler; import net.jrtechs.www.SteamAPI.APIConnection; import net.jrtechs.www.server.Player; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Queue; +import java.util.*; /** * Main class for digging up the entire * steam network. * - * @author Jeffery Russell + * @author Jeffery Russell 11-18-18 */ public class SteamWebCrawler { @@ -28,7 +25,17 @@ public class SteamWebCrawler private FileIO fileIO; /** Queue used for a BFS search */ - private Queue downlaodQueue; + private LinkedList downlaodQueue; + + /** Players which have been detected by + * our search and currently in a queue + * or has already been processed*/ + private HashSet visited; + + /** List of players which we have accessed + * in the steam network, but, have no clue what + * their name is*/ + private LinkedList namelessQueue; /** @@ -43,41 +50,92 @@ public class SteamWebCrawler this.fileIO = new FileIO("/media/jeff/A4BA9239BA920846/steamData/"); this.downlaodQueue = new LinkedList<>(); + + visited = new HashSet<>(); + + namelessQueue = new LinkedList<>(); } /** - * Runs a BFS search of the steam network + * If the download queue is empty, this will + * look up the names of the first 100 players in the + * nameless queue and add them to download queue. */ - private void runCrawler() + private void shiftNamelessToDownload() { - while(!downlaodQueue.isEmpty()) + if(this.downlaodQueue.isEmpty() && !this.namelessQueue.isEmpty()) { - Player current = downlaodQueue.remove(); - - List currentFriends = connection.getFriends(current.getId()); + List winners = new ArrayList<>(); + for(int i = 0; i < (100 < namelessQueue.size()? 100: namelessQueue.size()); i++) + { + winners.add(this.namelessQueue.remove()); + } + List namedPlayers = connection.getFullPlayers(winners); + this.throttler.wait(1); + downlaodQueue.addAll(namedPlayers); + } + } - List neededFriends = new ArrayList<>(); - currentFriends.forEach(s -> + /** + * Does one of the following three actions for each + * of the steam members in the list: + * 1: Ignore- already has been queued by program + * 2: Add to nameless queue -- doesn't have name yet + * 3: Add to download queue -- already on HHD but needed for + * the search algo to work. + * + * @param ids list of steam ids + */ + private void queueUpPlayers(List ids) + { + for(String s: ids) + { + if(!visited.contains(s)) { - if(!fileIO.playerExists(s)) - neededFriends.add(s); - }); + if(fileIO.playerExists(s)) + { + downlaodQueue.add(new Player("dummy", s)); + } + else + { + namelessQueue.add(s); + } + visited.add(s); + } + } + System.out.println("Download Queue: " + downlaodQueue.size()); + System.out.println("Nameless Queue: " + namelessQueue.size()); + } - connection.getFullPlayers(neededFriends).forEach(f-> - { - downlaodQueue.add(f); - }); - int queriesRan = neededFriends.size()/100 + 2; - this.throttler.wait(queriesRan); + /** + * Runs a BFS search of the steam network + */ + private void runCrawler() + { + while(!downlaodQueue.isEmpty()) + { + Player current = downlaodQueue.remove(); + + List currentFriends; if(!fileIO.playerExists(current.getId())) { + this.throttler.wait(1); + currentFriends = connection.getFriends(current.getId()); fileIO.writeToFile(current, currentFriends); } + else + { + currentFriends = fileIO.readFriends(current.getId()); + } + + queueUpPlayers(currentFriends); + + shiftNamelessToDownload(); } }