diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d5819e --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# Steam Graph Analysis + +This is a project that I threw together during the weekend to play around with +gremlin graph database. Currently this project scrapes the steam API for friends +and their friends which can be used to generate a graph. This information is stored +locally in a gremlin server and is then sent to the client via a web socket. + +![Diagram](Diagram.svg) + + +[Video Of Friends of Friends Graph](https://www.youtube.com/watch?v=WJfo9bU0nH8) + + +This project is in the VERY early stages of development and is far from finished. +If you are lucky, you will find it live at [http://jrtechs.student.rit.edu/friendsOfFriends.html](http://jrtechs.student.rit.edu/friendsOfFriends.html). +It is still being actively developed and does not have permanent hosting so there is a %60 +chance at any time that you will be able to access it. + + +# Bugs +* Does not work with firefox +* Tends to crash w/o telling user if you provide an invalid steam id + + +# TODO +* Include a steam name to steam id lookup +* Dockerize this entire environment +* Connect the gremlin/janus server to a HBase server for persistent storage +* Make the graphs look better -- possibly switch from sigma.js to d3 +* Get the java web socket to work with ssh -- currently does not work with wss +* Make more graphs to provide more insights + * Friends with friends -- shows which of your friends are friends with each other + * Most common friends friends -- will show you people you may know + * Graph of a larger chunk of the steam community + * ... +* Write more documentation on how the system as a whole works. +* Write blog post on what I learned during this project. \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java index f499402..2af5122 100644 --- a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java +++ b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java @@ -3,11 +3,13 @@ package net.jrtechs.www.SteamAPI; import net.jrtechs.www.utils.ConfigLoader; import net.jrtechs.www.utils.WebScraper; +import org.json.JSONArray; import org.json.JSONObject; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Class which is used to pull information from the Steam api @@ -73,6 +75,47 @@ public class APIConnection } + /** + * returns a map from the steam id to the players name + * + * * tricky because we can only request up to 100 ids + * in one request + * + * @param ids + * @return + */ + public Map getNames(List ids) + { + System.out.println(ids); + Map map = new HashMap<>(); + + while(!ids.isEmpty()) + { + String queryUrl = baseURL + playerInfoURL + apiKey + "&steamids="; + + int remove = (ids.size() > 100) ? 100 : ids.size(); + + for(int i = 0; i < remove; i++) + { + queryUrl = queryUrl + "," + ids.remove(0); + } + + System.out.println(queryUrl); + JSONArray names = new JSONObject(WebScraper.getWebsite(queryUrl)) + .getJSONObject("response").getJSONArray("players"); + + for(int i = 0; i < names.length(); i++) + { + JSONObject player = names.getJSONObject(i); + System.out.println(player); + map.put(player.getString("steamid"), + player.getString("personaname")); + } + } + return map; + } + + /** * Returns the name of the player with a specific steam id * @@ -83,7 +126,7 @@ public class APIConnection { return ((HashMap) new JSONObject(WebScraper .getWebsite(this.baseURL + this.playerInfoURL + - this.apiKey + "&steamids=" + steamid)) + this.apiKey + "&steamids=" + steamid)) .getJSONObject("response") .getJSONArray("players") .toList().stream().findAny().get()).get("personaname"); @@ -97,4 +140,4 @@ public class APIConnection System.out.println(con.getPlayerName("76561198188400721")); } -} +} \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/client/bestFriends.html b/src/main/java/net/jrtechs/www/client/bestFriends.html deleted file mode 100644 index e81c175..0000000 --- a/src/main/java/net/jrtechs/www/client/bestFriends.html +++ /dev/null @@ -1,136 +0,0 @@ - - - - - Jrtechs Steam Friend Graph Project - - - - - - - - - -
-


- -

TDLTR

- -
- - - - - -
- - -
- - - - - - - diff --git a/src/main/java/net/jrtechs/www/client/serverConnection.js b/src/main/java/net/jrtechs/www/client/serverConnection.js deleted file mode 100644 index 5d5588e..0000000 --- a/src/main/java/net/jrtechs/www/client/serverConnection.js +++ /dev/null @@ -1,56 +0,0 @@ -var connection = new WebSocket('ws://127.0.0.1:4444'); - -connection.onopen = function () -{ - console.log('Connected!'); - connection.send('Ping'); // Send the message 'Ping' to the server -}; - -// Log errors -connection.onerror = function (error) -{ - console.log('WebSocket Error ' + error); -}; - -function addNodeToGraph(request) -{ - s.graph.addNode({ - id: request.id, - label: request.name, - x: Math.random(), - y: Math.random(), - size: Math.random(), - color: '#666' - }); - s.refresh(); -} - - -function addEdgeToGraph(request) -{ - s.graph.addEdge({ - id: request.id, - source: request.p1, - target: request.p2, - size: Math.random(), - color: '#000' - }); - s.refresh(); -} - - -// Log messages from the server -connection.onmessage = function (e) -{ - var request = JSON.parse(e.data); - - if(request.action == 1) - { - addNodeToGraph(request); - } - else if(request.action == 2) - { - addEdgeToGraph(request); - } - console.log('Server: ' + e.data); -}; \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java index 954416a..1da5d23 100644 --- a/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java +++ b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java @@ -1,11 +1,12 @@ package net.jrtechs.www.graphDB; -import net.jrtechs.www.Player; +import net.jrtechs.www.server.Player; import net.jrtechs.www.SteamAPI.APIConnection; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Does graph based operations with {@link Player} @@ -42,9 +43,36 @@ public class SteamGraph private boolean alreadyInGraph(String id) { String query = "g.V().hasLabel('player')" + - ".has('id', '" + id + "')"; - System.out.println(query); - return (1 == con.queryGraph(query).stream().count()); + ".has('id', '" + id + "')"; + //System.out.println(query); + return (1 <= con.queryGraph(query).stream().count()); + } + + + + /** + * Inserts a player vertex into the graph + * + * @param + */ + private void insertPlayerIntoGraph(String id, String name) + { + try + { + if(!this.alreadyInGraph(id)) + { + String queryInsertPlayer = "g.addV('player')" + + ".property('name', '" + name + "')" + + ".property('crawled', '0')" + + ".property('id', '" + id + "')"; + System.out.println("inserting " + name + " into graph"); + this.con.queryGraph(queryInsertPlayer); + } + } + catch (Exception e) + { + e.printStackTrace(); + } } @@ -56,16 +84,16 @@ public class SteamGraph * @param p2 * @return */ - private boolean edgeAlreadyInGraph(Player p1, Player p2) + private boolean edgeAlreadyInGraph(String p1, String p2) { try { String query = "g.V().hasLabel('player')" + - ".has('id', '" + p1.getId() + "')" + + ".has('id', '" + p1 + "')" + ".both()" + - ".has('id', '" + p2.getId() + "')"; - System.out.println(query); - return (1 == con.queryGraph(query).stream().count()); + ".has('id', '" + p2 + "')"; + //System.out.println(query); + return (1 <= con.queryGraph(query).stream().count()); } catch(Exception e) { @@ -74,20 +102,6 @@ public class SteamGraph } - /** - * Inserts a player vertex into the graph - * - * @param player - */ - private void insertSinglePlayer(Player player) - { - String queryInsertPlayer = "g.addV('player')" + - ".property('name', '" + player.getName() + "')" + - ".property('id', '" + player.getId() + "')"; - System.out.println(queryInsertPlayer); - this.con.queryGraph(queryInsertPlayer); - } - /** * Inserts a edge between two players into the graph @@ -95,64 +109,69 @@ public class SteamGraph * @param p1 * @param p2 */ - private void insertEdge(Player p1, Player p2) + private void insertEdgeIntoGraph(String p1, String p2) { - String query = "g.V().hasLabel('player')" + - ".has('id', '" + p1.getId() + "')" + - ".as('p1')" + - "V().hasLabel('player')" + - ".has('id', '" + p2.getId() + "')" + - ".as('p2')" + - ".addE('friends')" + - ".from('p1').to('p2')"; - System.out.println(query); - this.con.queryGraph(query); + try + { + if(!this.edgeAlreadyInGraph(p1, p2)) + { + String query = "g.V().hasLabel('player')" + + ".has('id', '" + p1 + "')" + + ".as('p1')" + + "V().hasLabel('player')" + + ".has('id', '" + p2 + "')" + + ".as('p2')" + + ".addE('friends')" + + ".from('p1').to('p2')"; + //System.out.println(query); + this.con.queryGraph(query); + } + } + catch (Exception e) + { + e.printStackTrace(); + } + } /** - * Inserts a player and all of it's friends into - * the graph. + * determines if a player has been indexed for friends yet * - * @param player + * @param id + * @return */ - public void insertIntoGraph(Player player) + private boolean playerAlreadyIndexed(String id) { - System.out.println(player); - if(!this.alreadyInGraph(player.getId())) + try { - this.insertSinglePlayer(player); - } + String query = "g.V().hasLabel('player')" + + ".has('id', '" + id + "')" + + ".has('crawled', '0')"; - for(Player friend : player.fetchFriends(api)) + return (1 != con.queryGraph(query).stream().count()); + } + catch(Exception e) { - if(!alreadyInGraph(friend.getId())) - { - insertSinglePlayer(friend); - } - - if(!edgeAlreadyInGraph(player, friend)) - { - insertEdge(player, friend); - } + e.printStackTrace(); } + return true; } - /** - * Recursive function for scraping the steam api - * - * @param player - * @param debth - */ - public void insertIntoGraph(Player player, int debth) + private void updateCrawledStatus(String id) { - insertIntoGraph(player); + try + { + String query = "g.V().hasLabel('player')" + + ".has('id', '" + id + "')" + + ".property('crawled', '1')"; - if(debth > 0) + this.con.queryGraph(query); + } + catch (Exception e) { - player.fetchFriends(this.api) - .forEach(f -> insertIntoGraph(f, debth -1)); + e.printStackTrace(); } } @@ -188,39 +207,76 @@ public class SteamGraph ".both().valueMap()"; this.con.queryGraph(query).stream().forEach(r-> - friends.add(new Player( - ((ArrayList) (((HashMap)(r.getObject())) - .get("name"))).get(0).toString(), - ((ArrayList)(((HashMap)(r.getObject())) - .get("id"))).get(0).toString())) + friends.add(new Player( + ((ArrayList) (((HashMap)(r.getObject())) + .get("name"))).get(0).toString(), + ((ArrayList)(((HashMap)(r.getObject())) + .get("id"))).get(0).toString())) ); return friends; } + /** + * tells api to get this dude's friends list + * + * @param id + */ + private void indexPersonFriends(String id) + { + System.out.println("indexing " + this.getNameFromGraph(id)); + + List friendsIds = this.api.getFriends(id); + + //find ones not in database + List notInDatabase = new ArrayList<>(); + for(String fid : friendsIds) + { + if(!this.alreadyInGraph(fid)) + { + notInDatabase.add(fid); + } + } + Map names = this.api.getNames(notInDatabase); + + for(String key: names.keySet()) + { + this.insertPlayerIntoGraph(key, names.get(key)); + } + + friendsIds.forEach(s-> this.insertEdgeIntoGraph(id, s)); + + this.updateCrawledStatus(id); + } + /** - * Fetches a player from the graph + * Fetches a player from the graph with all of its friends * * @param id * @return */ - public Player getPlayerInformation(String id) + public Player getPlayer(String id) { Player p; - if(!this.alreadyInGraph(id)) + if(this.alreadyInGraph(id)) // yay { - p = new Player(id); - this.insertIntoGraph(p); + p = new Player(this.getNameFromGraph(id), id); + if(!this.playerAlreadyIndexed(id)) //must index the person + { + this.indexPersonFriends(id); + } + + p.setFriends(this.getFriendsFromGraph(id)); } - else + else //smh, shouldn't happen frequently { - p = new Player(this.getNameFromGraph(id), id); + System.out.println("brand spanking new request " + id); + String name = this.api.getPlayerName(id); + this.insertPlayerIntoGraph(id, name); + return this.getPlayer(id); } - - p.setFriends(this.getFriendsFromGraph(id)); - return p; } @@ -231,12 +287,12 @@ public class SteamGraph */ public static void main(String[] args) { - SteamGraph graph = new SteamGraph(); - - Player base = new Player(args[0]); - - int debth = Integer.valueOf(args[1]); - - graph.insertIntoGraph(base, debth); +// SteamGraph graph = new SteamGraph(); +// +// Player base = graph.getPlayer(args[0]); +// +// int debth = Integer.valueOf(args[1]); +// +// graph.insertIntoGraph(base, debth); } -} +} \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/server/Client.java b/src/main/java/net/jrtechs/www/server/Client.java index bdeaabb..fb13e33 100644 --- a/src/main/java/net/jrtechs/www/server/Client.java +++ b/src/main/java/net/jrtechs/www/server/Client.java @@ -1,6 +1,5 @@ package net.jrtechs.www.server; -import net.jrtechs.www.Player; import net.jrtechs.www.graphDB.SteamGraph; import org.java_websocket.WebSocket; import org.json.JSONObject; @@ -30,34 +29,24 @@ public class Client extends Thread private int debth; + private int type; + + /** * Initializes the client with a steam graph and * web socket information. * @param client */ - public Client(WebSocket client) + public Client(WebSocket client, String id, int type) { this.client = client; this.graph = new SteamGraph(); - - //temp stuff - this.baseId = "76561198176504246"; + this.type = type; + this.baseId = id; this.debth = 1; } - /** - * Method which is called when the client sends a message - * to the server. - * - * @param message - */ - public void receivedMessage(String message) - { - // we don't care about this yet - } - - /** * returns the web socket object * @@ -111,16 +100,14 @@ public class Client extends Thread private void sendJSON(JSONObject request) { - System.out.println("sending " + request.toString()); this.client.send(request.toString()); - try { Thread.sleep(50); //prevents DDOSing the client } catch (Exception e) { - + e.printStackTrace(); } } @@ -150,8 +137,6 @@ public class Client extends Thread this.sendEdgeAdd(p, friend); currentStep += radianStep; - - System.out.println(currentStep); } } @@ -162,7 +147,7 @@ public class Client extends Thread @Override public void run() { - Player b = this.graph.getPlayerInformation(this.baseId); + Player b = this.graph.getPlayer(this.baseId); List friends = b.fetchFriends(); this.sendPlayerToClient(b, 300, 243, 1); @@ -174,10 +159,12 @@ public class Client extends Thread for(Player f : b.fetchFriends()) { - f = this.graph.getPlayerInformation(f.getId()); + f = this.graph.getPlayer(f.getId()); this.sendPlayerToClient(f, (int)(300 + Math.cos(currentStep) * 300), (int)(243 + Math.sin(currentStep) * 300) ,2); currentStep += radianStep; } + + this.client.close(); } -} +} \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/Player.java b/src/main/java/net/jrtechs/www/server/Player.java similarity index 61% rename from src/main/java/net/jrtechs/www/Player.java rename to src/main/java/net/jrtechs/www/server/Player.java index 5d0de26..acc15a2 100644 --- a/src/main/java/net/jrtechs/www/Player.java +++ b/src/main/java/net/jrtechs/www/server/Player.java @@ -1,6 +1,4 @@ -package net.jrtechs.www; - -import net.jrtechs.www.SteamAPI.APIConnection; +package net.jrtechs.www.server; import java.util.ArrayList; import java.util.List; @@ -37,39 +35,6 @@ public class Player } - /** - * Sets the fields of the player only based on it's - * steam id - * - * @param id - */ - public Player(String id) - { - this.id = id; - this.friends = null; - this.name = new APIConnection().getPlayerName(id); - } - - - /** - * Returns a list of all the friends of a specific player - * - * @param con - * @return - */ - public List fetchFriends(APIConnection con) - { - if(this.friends == null) - { - this.friends = new ArrayList<>(); - con.getFriends(this.id) - .forEach(f-> this.friends.add( - new Player(con.getPlayerName(f), f))); - } - return friends; - } - - /** * Returns a list of all the friends of a specific player * @@ -112,4 +77,4 @@ public class Player { return "Name: " + this.name + " id: " + this.id; } -} +} \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/server/Server.java b/src/main/java/net/jrtechs/www/server/Server.java index 6657ebd..badf224 100644 --- a/src/main/java/net/jrtechs/www/server/Server.java +++ b/src/main/java/net/jrtechs/www/server/Server.java @@ -3,6 +3,7 @@ package net.jrtechs.www.server; import org.java_websocket.WebSocket; import org.java_websocket.handshake.ClientHandshake; import org.java_websocket.server.WebSocketServer; +import org.json.JSONObject; import java.net.InetSocketAddress; import java.util.HashSet; @@ -29,6 +30,8 @@ public class Server extends WebSocketServer public Server() { super(new InetSocketAddress(TCP_PORT)); + + System.out.println(super.getAddress()); clients = new HashSet<>(); } @@ -36,11 +39,6 @@ public class Server extends WebSocketServer @Override public void onOpen(WebSocket conn, ClientHandshake handshake) { - Client newClient = new Client(conn); - clients.add(newClient); - - newClient.start(); - System.out.println("New connection from " + conn.getRemoteSocketAddress().getAddress().getHostAddress()); } @@ -58,19 +56,34 @@ public class Server extends WebSocketServer public void onMessage(WebSocket conn, String message) { System.out.println("Message from client: " + message); - for (Client client : clients) + + JSONObject object = new JSONObject(message); + System.out.println(message); + + if(object.has("graph")) { - if(client.getSocket() == conn) + for (Client client : clients) { - client.receivedMessage(message); + if(client.getSocket() == conn) + { + return; + } } + + Client newClient = new Client(conn, object.getString("id"), + object.getInt("graph")); + clients.add(newClient); + + newClient.start(); } + + } @Override public void onError(WebSocket conn, Exception ex) { - //ex.printStackTrace(); + ex.printStackTrace(); if (conn != null) { clients.remove(conn); @@ -110,4 +123,4 @@ public class Server extends WebSocketServer new Server().start(); } -} +} \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/client/closestPath.html b/website/closestPath.html similarity index 95% rename from src/main/java/net/jrtechs/www/client/closestPath.html rename to website/closestPath.html index e81c175..3968a46 100644 --- a/src/main/java/net/jrtechs/www/client/closestPath.html +++ b/website/closestPath.html @@ -6,12 +6,11 @@ -