From b0a31138f85119b1ab1ffc349dfb8587903899d6 Mon Sep 17 00:00:00 2001 From: jrtechs Date: Fri, 10 Jul 2020 18:54:55 -0400 Subject: [PATCH] Migrated to using a Berkely backend --- conf/graph.properties | 6 + .../jrtechs/www/SteamAPI/APIConnection.java | 1 + .../jrtechs/www/graphDB/GraphConnection.java | 40 ++++ .../jrtechs/www/graphDB/GremlinConsole.java | 58 ++--- .../jrtechs/www/graphDB/RemoteConnection.java | 203 +++++++++--------- .../net/jrtechs/www/graphDB/SteamGraph.java | 134 +++++++----- .../java/net/jrtechs/www/server/Player.java | 10 +- 7 files changed, 265 insertions(+), 187 deletions(-) create mode 100644 conf/graph.properties create mode 100644 src/main/java/net/jrtechs/www/graphDB/GraphConnection.java diff --git a/conf/graph.properties b/conf/graph.properties new file mode 100644 index 0000000..f1b1c2d --- /dev/null +++ b/conf/graph.properties @@ -0,0 +1,6 @@ +gremlin.graph=org.janusgraph.core.JanusGraphFactory + +storage.backend=berkeleyje + +storage.directory=./steam-graph/db +query.smart-limit=false \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java index eaf1b53..eb177b5 100644 --- a/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java +++ b/src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java @@ -128,6 +128,7 @@ public class APIConnection return friendsId; //private url JSONObject object = new JSONObject(apiData); + System.out.println(object); if(object.has("friendslist")) { diff --git a/src/main/java/net/jrtechs/www/graphDB/GraphConnection.java b/src/main/java/net/jrtechs/www/graphDB/GraphConnection.java new file mode 100644 index 0000000..c2afe49 --- /dev/null +++ b/src/main/java/net/jrtechs/www/graphDB/GraphConnection.java @@ -0,0 +1,40 @@ +package net.jrtechs.www.graphDB; + + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.janusgraph.core.JanusGraphFactory; + +/** + * @author Jeffery Russell 7-10-2020 + */ +public class GraphConnection +{ + private Graph graph; + + private GraphTraversalSource t; + + public GraphConnection() + { + this.graph = JanusGraphFactory.open("conf/graph.properties"); + t = graph.traversal(); + } + + + public void closeConnection() throws Exception + { + this.commit(); + this.graph.close(); + } + + public void commit() + { + this.t.tx().commit(); + } + + + public GraphTraversalSource getTraversal() + { + return this.t; + } +} diff --git a/src/main/java/net/jrtechs/www/graphDB/GremlinConsole.java b/src/main/java/net/jrtechs/www/graphDB/GremlinConsole.java index 7de16e0..646f7e2 100644 --- a/src/main/java/net/jrtechs/www/graphDB/GremlinConsole.java +++ b/src/main/java/net/jrtechs/www/graphDB/GremlinConsole.java @@ -1,6 +1,7 @@ package net.jrtechs.www.graphDB; -import org.apache.tinkerpop.gremlin.driver.ResultSet; + +import org.apache.tinkerpop.gremlin.process.remote.RemoteConnection; import java.io.BufferedReader; import java.io.IOException; @@ -17,7 +18,7 @@ import java.util.stream.IntStream; public class GremlinConsole { /** Connection to graph server **/ - private RemoteConnection connection; + private GraphConnection connection; /** @@ -25,7 +26,7 @@ public class GremlinConsole */ public GremlinConsole() { - this.connection = new RemoteConnection(); + this.connection = new GraphConnection(); } @@ -34,7 +35,7 @@ public class GremlinConsole * * @return */ - public RemoteConnection getConnection() + public GraphConnection getConnection() { return this.connection; } @@ -68,15 +69,17 @@ public class GremlinConsole System.exit(0); } - ResultSet set = this.connection.queryGraph(input); - try - { - set.forEach(System.out::println); - } - catch (Exception ex) - { - ex.printStackTrace(); - } + + +// ResultSet set = this.connection.queryGraph(input); +// try +// { +// set.forEach(System.out::println); +// } +// catch (Exception ex) +// { +// ex.printStackTrace(); +// } } @@ -113,17 +116,24 @@ public class GremlinConsole * * @param args */ - public static void main(String args[]) + public static void main(String args[]) throws Exception { - GremlinConsole console = new GremlinConsole(); - - //don't worry about this lambda - IntStream.range(0, args.length) - .forEach(i-> console.getConnection() - .queryGraph(args[i]) - .forEach((System.out::println))); - - console.run(); +// GremlinConsole console = new GremlinConsole(); +// +// //don't worry about this lambda +// IntStream.range(0, args.length) +// .forEach(i-> console.getConnection() +// .queryGraph(args[i]) +// .forEach((System.out::println))); +// +// console.run(); + + GraphConnection con = new GraphConnection(); + + System.out.println(con.getTraversal().E().toList() + ); + System.out.println(con.getTraversal().V().toList() + ); + con.closeConnection(); } - } diff --git a/src/main/java/net/jrtechs/www/graphDB/RemoteConnection.java b/src/main/java/net/jrtechs/www/graphDB/RemoteConnection.java index c61dc40..fc188fe 100644 --- a/src/main/java/net/jrtechs/www/graphDB/RemoteConnection.java +++ b/src/main/java/net/jrtechs/www/graphDB/RemoteConnection.java @@ -1,101 +1,102 @@ -package net.jrtechs.www.graphDB; - -import org.apache.tinkerpop.gremlin.driver.Client; -import org.apache.tinkerpop.gremlin.driver.Cluster; -import org.apache.tinkerpop.gremlin.driver.ResultSet; - -import net.jrtechs.www.utils.ConfigLoader; - -/** - * Simple helper class which allows us to remotely connect to a - * remote graph - * - * @author Jeffery Russell 5-24-18 - */ -public class RemoteConnection -{ - - /** Stores/manages client connections **/ - private Cluster cluster; - - - /** - * Connection to the graph db - */ - private Client client; - - - /** - * Connects to a remote Graph database - * - * Check this link out to learn about Cluster.Builder() - * http://tinkerpop.apache.org/javadocs/3.3.3/core/org/apache/tinkerpop/ - * gremlin/driver/Cluster.Builder.html - * - */ - public RemoteConnection() - { - ConfigLoader conf = new ConfigLoader("GremlinServerConnection.json"); - - Cluster.Builder b = Cluster.build(); - b.addContactPoint(conf.getValue("host")); - b.port(conf.getInt("port")); - - b.credentials(conf.getValue("username"), conf.getValue("password")); - - this.cluster = b.create(); - - this.client = cluster.connect(); - } - - - /** - * Queries the graph and return the results which can be iterated over - * - * ex: - * ResultSet results = remote.queryGraph("g.V().values('name')"); - * - * results.stream().forEach(result -> - * { - * String s = result.getString(); - * System.out.println("name: " + s); - * }); - * - * @param q - * @return - */ - public ResultSet queryGraph(String q) - { - return this.client.submit(q); - } - - - /** - * Closes connection with remote database - */ - public void closeConnection() - { - this.cluster.close(); - } - - - /** - * testing method which will be removed soon - * @param args - */ - public static void main(String args[]) - { - RemoteConnection remote = new RemoteConnection(); - ResultSet results = remote.queryGraph("g.V().values('name')"); - - //results.stream().forEach(System.out::println); - - results.stream().forEach(result -> - { - String s = result.getString(); - System.out.println("name: " + s); - }); - - remote.closeConnection(); - } -} \ No newline at end of file +//package net.jrtechs.www.graphDB; +// +//import org.apache.tinkerpop.gremlin.driver.Client; +//import org.apache.tinkerpop.gremlin.driver.Cluster; +//import org.apache.tinkerpop.gremlin.driver.ResultSet; +// +//import net.jrtechs.www.utils.ConfigLoader; +// +///** +// * Simple helper class which allows us to remotely connect to a +// * remote graph +// * +// * @author Jeffery Russell 5-24-18 +// */ +//public class RemoteConnection +//{ +// +// /** Stores/manages client connections **/ +// private Cluster cluster; +// +// +// /** +// * Connection to the graph db +// */ +// private Client client; +// +// +// /** +// * Connects to a remote Graph database +// * +// * Check this link out to learn about Cluster.Builder() +// * http://tinkerpop.apache.org/javadocs/3.3.3/core/org/apache/tinkerpop/ +// * gremlin/driver/Cluster.Builder.html +// * +// */ +// public RemoteConnection() +// { +// ConfigLoader conf = new ConfigLoader("GremlinServerConnection.json"); +// +// Cluster.Builder b = Cluster.build(); +// b.addContactPoint(conf.getValue("host")); +// b.port(conf.getInt("port")); +// +// b.credentials(conf.getValue("username"), conf.getValue("password")); +// +// this.cluster = b.create(); +// +// this.client = cluster.connect(); +// } +// +// +// /** +// * Queries the graph and return the results which can be iterated over +// * +// * ex: +// * ResultSet results = remote.queryGraph("g.V().values('name')"); +// * +// * results.stream().forEach(result -> +// * { +// * String s = result.getString(); +// * System.out.println("name: " + s); +// * }); +// * +// * @param q +// * @return +// */ +// public ResultSet queryGraph(String q) +// { +// return this.client.submit(q); +// } +// +// +// /** +// * Closes connection with remote database +// */ +// public void closeConnection() +// { +// this.cluster.close(); +// } +// +// +// /** +// * testing method which will be removed soon +// * @param args +// */ +// public static void main(String args[]) +// { +// +// RemoteConnection remote = new RemoteConnection(); +// ResultSet results = remote.queryGraph("g.V().values('name')"); +// +// //results.stream().forEach(System.out::println); +// +// results.stream().forEach(result -> +// { +// String s = result.getString(); +// System.out.println("name: " + s); +// }); +// +// remote.closeConnection(); +// } +//} \ No newline at end of file diff --git a/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java index c070857..e63a546 100644 --- a/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java +++ b/src/main/java/net/jrtechs/www/graphDB/SteamGraph.java @@ -10,14 +10,14 @@ import java.util.Map; /** * Does graph based operations with {@link Player} - * and {@link RemoteConnection} + * and * * @author Jeffery Russell 5-26-17 */ public class SteamGraph { /** Connection to the graph server */ - private RemoteConnection con; + private GraphConnection con; /** Connection to steam api */ private APIConnection api; @@ -29,7 +29,7 @@ public class SteamGraph */ public SteamGraph() { - this.con = new RemoteConnection(); + this.con = new GraphConnection(); this.api = new APIConnection(); } @@ -42,10 +42,11 @@ public class SteamGraph */ private boolean alreadyInGraph(String id) { - String query = "g.V().hasLabel('player')" + - ".has('id', '" + id + "')"; - //System.out.println(query); - return (1 <= con.queryGraph(query).stream().count()); + System.out.println("Checking id:" + id); + return !con.getTraversal() + .V().hasLabel("player") + .has("id", id) + .toList().isEmpty(); } @@ -55,18 +56,18 @@ public class SteamGraph * * @param */ - private void insertPlayerIntoGraph(String id, String name) + private void insertPlayerIntoGraph(String id, String name, boolean check) { try { - if(!this.alreadyInGraph(id)) + if(!check || !this.alreadyInGraph(id)) { - String queryInsertPlayer = "g.addV('player')" + - ".property('name', '" + name + "')" + - ".property('crawled', '0')" + - ".property('id', '" + id + "')"; System.out.println("inserting " + name + " into graph"); - this.con.queryGraph(queryInsertPlayer); + this.con.getTraversal() + .addV("player") + .property("name", name) + .property("crawled", 0) + .property("id", id).id().next(); } } catch (Exception e) @@ -88,12 +89,12 @@ public class SteamGraph { try { - String query = "g.V().hasLabel('player')" + - ".has('id', '" + p1 + "')" + - ".both()" + - ".has('id', '" + p2 + "')"; - //System.out.println(query); - return (1 <= con.queryGraph(query).stream().count()); + return !this.con.getTraversal() + .V().hasLabel("player") + .has("id", p1) + .both() + .has("id", p2) + .toList().isEmpty(); } catch(Exception e) { @@ -111,20 +112,22 @@ public class SteamGraph */ private void insertEdgeIntoGraph(String p1, String p2) { + try { if(!this.edgeAlreadyInGraph(p1, p2)) { - String query = "g.V().hasLabel('player')" + - ".has('id', '" + p1 + "')" + - ".as('p1')" + - "V().hasLabel('player')" + - ".has('id', '" + p2 + "')" + - ".as('p2')" + - ".addE('friends')" + - ".from('p1').to('p2')"; - //System.out.println(query); - this.con.queryGraph(query); + System.out.println("Inserting edge: " + p1 + ":" + p2); + this.con.getTraversal() + .V() + .hasLabel("player") + .has("id", p1) + .as("p1") + .V().hasLabel("player") + .has("id", p2) + .as("p2") + .addE("friends") + .from("p1").to("p2").id().next(); } } catch (Exception e) @@ -145,11 +148,11 @@ public class SteamGraph { try { - String query = "g.V().hasLabel('player')" + - ".has('id', '" + id + "')" + - ".has('crawled', '0')"; - - return (1 != con.queryGraph(query).stream().count()); + return this.con.getTraversal() + .V().hasLabel("player") + .has("id", id) + .has("crawled", 0) + .toList().isEmpty(); } catch(Exception e) { @@ -163,11 +166,10 @@ public class SteamGraph { try { - String query = "g.V().hasLabel('player')" + - ".has('id', '" + id + "')" + - ".property('crawled', '1')"; - - this.con.queryGraph(query); + this.con.getTraversal().V() + .hasLabel("player") + .has("id", id) + .property("crawled", 1).id().next(); } catch (Exception e) { @@ -184,11 +186,11 @@ public class SteamGraph */ private String getNameFromGraph(String id) { - String query = "g.V().hasLabel('player')" + - ".has('id', '" + id + "')" + - ".values('name')"; - return this.con.queryGraph(query).stream() - .findFirst().get().getObject().toString(); + return this.con.getTraversal().V() + .hasLabel("player") + .has("id", id) + .values("name") + .toStream().findFirst().get().toString(); } @@ -200,19 +202,19 @@ public class SteamGraph */ private List getFriendsFromGraph(String id) { + System.out.println("fetching friends from graph"); List friends = new ArrayList<>(); - - String query = "g.V().hasLabel('player')" + - ".has('id', '" + id + "')" + - ".both().valueMap()"; try { - this.con.queryGraph(query).stream().forEach(r -> - friends.add(new Player( - ((ArrayList) (((HashMap) (r.getObject())) - .get("name"))).get(0).toString(), - ((ArrayList) (((HashMap) (r.getObject())) - .get("id"))).get(0).toString())) + this.con.getTraversal().V() + .hasLabel("player") + .has("id", id) + .both().valueMap().toStream().forEach(r -> + friends.add( + new Player(r.get("name").toString(), + r.get("id").toString() + ) + ) ); } catch(Exception e) @@ -247,12 +249,13 @@ public class SteamGraph for(String key: names.keySet()) { - this.insertPlayerIntoGraph(key, names.get(key)); + this.insertPlayerIntoGraph(key, names.get(key), false); } friendsIds.forEach(s-> this.insertEdgeIntoGraph(id, s)); this.updateCrawledStatus(id); + this.con.commit(); } @@ -285,13 +288,25 @@ public class SteamGraph } else { - this.insertPlayerIntoGraph(id, name); + this.insertPlayerIntoGraph(id, name, false); return this.getPlayer(id); } } return p; } + public void close() + { + try + { + this.con.closeConnection(); + } + catch (Exception e) + { + e.printStackTrace(); + } + } + /** * @@ -299,7 +314,10 @@ public class SteamGraph */ public static void main(String[] args) { -// SteamGraph graph = new SteamGraph(); + SteamGraph graph = new SteamGraph(); + graph.getPlayer("76561198013779806").getFriends().stream().forEach(System.out::println); +// graph.indexPersonFriends("76561198188400721"); + graph.close(); // // Player base = graph.getPlayer(args[0]); // diff --git a/src/main/java/net/jrtechs/www/server/Player.java b/src/main/java/net/jrtechs/www/server/Player.java index e92b466..49e1e42 100644 --- a/src/main/java/net/jrtechs/www/server/Player.java +++ b/src/main/java/net/jrtechs/www/server/Player.java @@ -38,15 +38,17 @@ public class Player this.id = id; this.friends = null; this.date = new Date(); - + this.friends = new ArrayList<>(); } - public List getFriends() { + public List getFriends() + { return friends; } - public Date getDate() { + public Date getDate() + { return date; } @@ -90,6 +92,6 @@ public class Player @Override public String toString() { - return "Name: " + this.name + " id: " + this.id; + return "Name: " + this.name + " id: " + this.id + " " + friends.size(); } } \ No newline at end of file