From 85a2f8d6bcdc9114cd94fc5a20b5bb7bd6693ef3 Mon Sep 17 00:00:00 2001 From: jrtechs Date: Sun, 18 Nov 2018 19:40:20 -0500 Subject: [PATCH] Created the basic file structure for crawling the steam friends network. --- .../net/jrtechs/www/graphDB/SteamGraph.java | 4 +- .../java/net/jrtechs/www/server/Player.java | 15 +++++ .../jrtechs/www/webCrawler/APIThrottler.java | 28 +++++++++ .../net/jrtechs/www/webCrawler/FileIO.java | 59 +++++++++++++++++++ .../www/webCrawler/SteamWebCrawler.java | 34 +++++++++++ .../www/webCrawler/SteamdFileWriter.java | 27 +++++++++ 6 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java create mode 100644 src/main/java/net/jrtechs/www/webCrawler/FileIO.java create mode 100644 src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java create mode 100644 src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java diff --git a/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java index 13b8285..c070857 100644 --- a/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java +++ b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java @@ -195,8 +195,8 @@ public class SteamGraph /** * Fetches a list of friends from the graph database * - * @param id - * @return + * @param id steam id + * @return list of friends */ private List getFriendsFromGraph(String id) { diff --git a/src/main/java/net/jrtechs/www/server/Player.java b/src/main/java/net/jrtechs/www/server/Player.java index acc15a2..e92b466 100644 --- a/src/main/java/net/jrtechs/www/server/Player.java +++ b/src/main/java/net/jrtechs/www/server/Player.java @@ -1,6 +1,8 @@ package net.jrtechs.www.server; +import java.time.LocalDate; import java.util.ArrayList; +import java.util.Date; import java.util.List; @@ -20,6 +22,9 @@ public class Player /** List of friends the player has */ private List friends; + /** Time which the player was crawled */ + private Date date; + /** * Sets the name and id of the player @@ -32,8 +37,18 @@ public class Player this.name = name; this.id = id; this.friends = null; + this.date = new Date(); + + } + + + public List getFriends() { + return friends; } + public Date getDate() { + return date; + } /** * Returns a list of all the friends of a specific player diff --git a/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java b/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java new file mode 100644 index 0000000..8e6d7ac --- /dev/null +++ b/src/main/java/net/jrtechs/www/webCrawler/APIThrottler.java @@ -0,0 +1,28 @@ +package net.jrtechs.www.webCrawler; + +import java.util.Calendar; + +/** + * @author Jeffery Russell + */ +public class APIThrottler +{ + public int totalqueries; + + private long lastQuery; + + + boolean queryAvailable() + { + return true; + } + + + public long getCurrentTimeInMS() + { + Calendar calendar = Calendar.getInstance(); + //Returns current time in millis + long timeMilli2 = calendar.getTimeInMillis(); + return timeMilli2; + } +} diff --git a/src/main/java/net/jrtechs/www/webCrawler/FileIO.java b/src/main/java/net/jrtechs/www/webCrawler/FileIO.java new file mode 100644 index 0000000..725d37b --- /dev/null +++ b/src/main/java/net/jrtechs/www/webCrawler/FileIO.java @@ -0,0 +1,59 @@ +package net.jrtechs.www.webCrawler; + +import com.google.gson.Gson; +import net.jrtechs.www.server.Player; + + +/** + * File which handles the file IO for storing + * all the players on the HHD + * + * @author Jeffery Russell 11-18-18 + */ +public class FileIO +{ + /** Base directory to store all the data */ + private String baseFilaPath; + + + /** Object used to convert objects to json strings */ + private final Gson gson; + + /** + * Initalizes the base directory + * @param basePath + */ + public FileIO(String basePath) + { + this.baseFilaPath = basePath; + this.gson = new Gson(); + } + + + /** + * Determines if we already have the player + * on disk. + * + * @param id + * @return + */ + public boolean playerExists(String id) + { + return false; + } + + + /** + * Writes the player to the file. + * + * @param player + */ + public void writeToFile(Player player) + { + String data = gson.toJson(player); + + String fileName = baseFilaPath + player.getId() + ".json"; + + SteamdFileWriter.writeToFile(data, fileName); + } +} diff --git a/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java b/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java new file mode 100644 index 0000000..a63320a --- /dev/null +++ b/src/main/java/net/jrtechs/www/webCrawler/SteamWebCrawler.java @@ -0,0 +1,34 @@ +package net.jrtechs.www.webCrawler; + +import net.jrtechs.www.SteamAPI.APIConnection; + +import java.io.File; + + +/** + * Main class for digging up the entire + * steam network. + * + * @author Jeffery Russell + */ +public class SteamWebCrawler +{ + + private APIThrottler throttler; + + private APIConnection connection; + + private FileIO fileIO; + + public void runSteamCrawler(String baseID) + { + + } + + + + public static void main(String args[]) + { + new SteamWebCrawler().runSteamCrawler("76561198188400721"); + } +} diff --git a/src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java b/src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java new file mode 100644 index 0000000..318fa3b --- /dev/null +++ b/src/main/java/net/jrtechs/www/webCrawler/SteamdFileWriter.java @@ -0,0 +1,27 @@ +package net.jrtechs.www.webCrawler; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; + +public class SteamdFileWriter +{ + public static void writeToFile(String data, String fileName) + { + BufferedWriter writer; + try + { + File file = new File(fileName); + file.createNewFile(); + writer = new BufferedWriter(new FileWriter(file)); + writer.write(data); + writer.flush(); + writer.close(); + System.out.println("Wrote to " + fileName); + } + catch (Exception e) + { + e.printStackTrace(); + } + } +}