From 3dceab620e81c003e37cab618c1a737da73d2231 Mon Sep 17 00:00:00 2001 From: jrtechs Date: Sat, 26 May 2018 14:00:13 -0400 Subject: [PATCH] Created a class which lets me scrape data and insert it into the graph database. --- src/main/java/net/jrtechs/www/Player.java | 100 +++++++++++ .../jrtechs/www/SteamAPI/APIConnection.java | 24 ++- .../net/jrtechs/www/graphDB/SteamGraph.java | 155 ++++++++++++++++++ .../net/jrtechs/www/utils/WebScraper.java | 26 ++- 4 files changed, 289 insertions(+), 16 deletions(-) create mode 100644 src/main/java/net/jrtechs/www/Player.java create mode 100644 src/main/java/net/jrtechs/www/graphDB/SteamGraph.java diff --git a/src/main/java/net/jrtechs/www/Player.java b/src/main/java/net/jrtechs/www/Player.java new file mode 100644 index 0000000..da3e055 --- /dev/null +++ b/src/main/java/net/jrtechs/www/Player.java @@ -0,0 +1,100 @@ +package net.jrtechs.www; + +import net.jrtechs.www.SteamAPI.APIConnection; + +import java.util.ArrayList; +import java.util.List; + + +/** + * Class to store information on a player + * + * @author Jeffery Russell 5-26-18 + */ +public class Player +{ + /** Name of the player **/ + private String name; + + /** Steam id of the player **/ + private String id; + + /** List of friends the player has */ + private List friends; + + + /** + * Sets the name and id of the player + * + * @param name + * @param id + */ + public Player(String name, String id) + { + this.name = name; + this.id = id; + this.friends = null; + } + + + /** + * Sets the fields of the player only based on it's + * steam id + * + * @param id + */ + public Player(String id) + { + this.id = id; + this.friends = null; + this.name = new APIConnection().getPlayerName(id); + } + + + /** + * Returns a list of all the friends of a specific player + * + * @param con + * @return + */ + public List fetchFriends(APIConnection con) + { + if(this.friends == null) + { + this.friends = new ArrayList<>(); + con.getFriends(this.id) + .forEach(f-> this.friends.add( + new Player(con.getPlayerName(f), f))); + } + + return friends; + } + + /** + * Getter for display name of player + * + * @return + */ + public String getName() + { + return this.name.replace("'", ""); + } + + + /** + * Getter for id of player + * + * @return + */ + public String getId() + { + return this.id; + } + + + @Override + public String toString() + { + return "Name: " + this.name + " id: " + this.id; + } +} diff --git a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java index 03c6060..f499402 100644 --- a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java +++ b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java @@ -53,14 +53,22 @@ public class APIConnection { List friendsId = new ArrayList<>(); - new JSONObject(WebScraper - .getWebsite(this.baseURL + this.friendListURL + - this.apiKey + "&steamid=" + steamid)) - .getJSONObject("friendslist") - .getJSONArray("friends").toList() - .forEach(f-> - friendsId.add(((HashMap)(f)).get("steamid")) - ); + try + { + new JSONObject(WebScraper + .getWebsite(this.baseURL + this.friendListURL + + this.apiKey + "&steamid=" + steamid)) + .getJSONObject("friendslist") + .getJSONArray("friends").toList() + .forEach(f-> + friendsId.add(((HashMap)(f)).get("steamid")) + ); + } + catch (Exception ex) + { + ex.printStackTrace(); + } + return friendsId; } diff --git a/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java new file mode 100644 index 0000000..1a6f5e4 --- /dev/null +++ b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java @@ -0,0 +1,155 @@ +package net.jrtechs.www.graphDB; + +import net.jrtechs.www.Player; +import net.jrtechs.www.SteamAPI.APIConnection; + +/** + * Does graph based operations with {@link Player} + * and {@link RemoteConnection} + * + * @author Jeffery Russell 5-26-17 + */ +public class SteamGraph +{ + /** Connection to the graph server */ + private RemoteConnection con; + + /** Connection to steam api */ + private APIConnection api; + + + /** + * Constructs object with a graph connection + * and a steam api connection + */ + public SteamGraph() + { + this.con = new RemoteConnection(); + this.api = new APIConnection(); + } + + + /** + * Checks if a player is already in the graph + * + * @param id steam id of player + * @return + */ + private boolean alreadyInGraph(String id) + { + String query = "g.V().hasLabel('player')" + + ".has('id', '" + id + "')"; + System.out.println(query); + return (1 == con.queryGraph(query).stream().count()); + } + + + /** + * Checks if a friend-friend edge is already in the + * graph + * + * @param p1 + * @param p2 + * @return + */ + private boolean edgeAlreadyInGraph(Player p1, Player p2) + { + String query = "g.V().hasLabel('player')" + + ".has('id', '" + p1.getId() + "')" + + ".both()" + + ".has('id', '" + p2.getId() + "')"; + System.out.println(query); + return (1 == con.queryGraph(query).stream().count()); + } + + /** + * Inserts a player vertex into the graph + * + * @param player + */ + private void insertSinglePlayer(Player player) + { + String queryInsertPlayer = "g.addV('player')" + + ".property('name', '" + player.getName() + "')" + + ".property('id', '" + player.getId() + "')"; + System.out.println(queryInsertPlayer); + this.con.queryGraph(queryInsertPlayer); + } + + + /** + * Inserts a edge between two players into the graph + * + * @param p1 + * @param p2 + */ + private void insertEdge(Player p1, Player p2) + { + String query = "g.V().hasLabel('player')" + + ".has('id', '" + p1.getId() + "')" + + ".as('p1')" + + "V().hasLabel('player')" + + ".has('id', '" + p2.getId() + "')" + + ".as('p2')" + + ".addE('friends')" + + ".from('p1').to('p2')"; + System.out.println(query); + this.con.queryGraph(query); + } + + + /** + * Inserts a player and all of it's friends into + * the graph. + * + * @param player + */ + public void insertIntoGraph(Player player) + { + System.out.println(player); + if(!this.alreadyInGraph(player.getId())) + { + this.insertSinglePlayer(player); + } + + for(Player friend : player.fetchFriends(api)) + { + if(!alreadyInGraph(friend.getId())) + { + insertSinglePlayer(friend); + } + + if(!edgeAlreadyInGraph(player, friend)) + { + insertEdge(player, friend); + } + } + } + + public void insertIntoGraph(Player player, int debth) + { + insertIntoGraph(player); + + if(debth > 0) + { + player.fetchFriends(this.api) + .forEach(f -> insertIntoGraph(f, debth -1)); + } + } + + + /** + * + * @param args + */ + public static void main(String[] args) + { + SteamGraph graph = new SteamGraph(); + + Player base = new Player(args[0]); + + int debth = Integer.valueOf(args[1]); + + graph.insertIntoGraph(base, debth); + } +} diff --git a/src/main/java/net/jrtechs/www/utils/WebScraper.java b/src/main/java/net/jrtechs/www/utils/WebScraper.java index 1a8d90e..2d955f9 100644 --- a/src/main/java/net/jrtechs/www/utils/WebScraper.java +++ b/src/main/java/net/jrtechs/www/utils/WebScraper.java @@ -9,22 +9,27 @@ import java.net.URL; * * @author Jeffery Russell 5-25-18 */ -public class WebScraper { +public class WebScraper +{ /** * Grabs the contents of a website as a string * * @param link to open * @return source code of website as a single string */ - public static String getWebsite(String link) { - try { + public static String getWebsite(String link) + { + try + { URL url = new URL(link); BufferedReader br = new BufferedReader( new InputStreamReader(url.openStream()) ); return WebScraper.getBufferedReaderData(br); - } catch (Exception e) { + } + catch (Exception e) + { e.printStackTrace(); } return ""; @@ -38,8 +43,10 @@ public class WebScraper { * @param postData to send * @return source code of website as a string */ - public static String getPostResponse(String link, String postData) { - try { + public static String getPostResponse(String link, String postData) + { + try + { URL url = new URL(link); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setDoOutput(true); @@ -53,7 +60,9 @@ public class WebScraper { new InputStreamReader(conn.getInputStream()) ); return WebScraper.getBufferedReaderData(br); - } catch (Exception ex) { + } + catch (Exception ex) + { ex.printStackTrace(); } @@ -73,7 +82,8 @@ public class WebScraper { throws IOException { String html = ""; String line; - while ((line = br.readLine()) != null) { + while ((line = br.readLine()) != null) + { html += line; } return html;