package net.jrtechs.www.webCrawler; import net.jrtechs.www.SteamAPI.APIConnection; import net.jrtechs.www.model.Player; import java.util.*; /** * Main class for digging up the entire * steam network. * * @author Jeffery Russell 11-18-18 */ @Deprecated public class SteamWebCrawler { /** Object used to limit the speed at which I access the steam * network */ private APIThrottler throttler; /** Connection to the steam network */ private APIConnection connection; /** Saves players to the disk */ private FileIO fileIO; /** Queue used for a BFS search */ private LinkedList downlaodQueue; /** Players which have been detected by * our search and currently in a queue * or has already been processed*/ private HashSet visited; /** List of players which we have accessed * in the steam network, but, have no clue what * their name is*/ private LinkedList namelessQueue; /** * Initializes the steam crawler's objects */ public SteamWebCrawler() { throttler = new APIThrottler(); this.connection = new APIConnection(); this.fileIO = new FileIO("/media/jeff/A4BA9239BA920846/steamData/"); this.downlaodQueue = new LinkedList<>(); visited = new HashSet<>(); namelessQueue = new LinkedList<>(); } /** * If the download queue is empty, this will * look up the names of the first 100 players in the * nameless queue and add them to download queue. */ private void shiftNamelessToDownload() { //this is a while instead of if because the getfull players query fails //once in a blue moon while(this.downlaodQueue.isEmpty() && !this.namelessQueue.isEmpty()) { List winners = new ArrayList<>(); for(int i = 0; i < (100 < namelessQueue.size()? 100: namelessQueue.size()); i++) { winners.add(this.namelessQueue.remove()); } List namedPlayers = connection.getPlayers(winners); this.throttler.wait(1); downlaodQueue.addAll(namedPlayers); } } /** * Does one of the following three actions for each * of the steam members in the list: * 1: Ignore- already has been queued by program * 2: Add to nameless queue -- doesn't have name yet * 3: Add to download queue -- already on HHD but needed for * the search algo to work. * * @param ids list of steam ids */ private void queueUpPlayers(List ids) { for(String s: ids) { if(!visited.contains(s)) { if(fileIO.playerExists(s)) { //downlaodQueue.add(new Player("dummy", s)); } else { namelessQueue.add(s); } visited.add(s); } } System.out.println("Download Queue: " + downlaodQueue.size()); System.out.println("Nameless Queue: " + namelessQueue.size()); } /** * Runs a BFS search of the steam network */ private void runCrawler() { while(!downlaodQueue.isEmpty()) { Player current = downlaodQueue.remove(); List currentFriends; if(!fileIO.playerExists(current.getId())) { this.throttler.wait(1); currentFriends = connection.getFriends(current.getId()); fileIO.writeToFile(current, currentFriends); } else { currentFriends = fileIO.readFriends(current.getId()); } queueUpPlayers(currentFriends); shiftNamelessToDownload(); } } /** * pop first fiend on the queue * and release the beast * * @param baseID */ public void runSteamCrawlerBase(String baseID) { //downlaodQueue.add(new Player("jrtechs", baseID)); runCrawler(); } public static void main(String args[]) { new SteamWebCrawler().runSteamCrawlerBase("76561198188400721"); } }