Merge pull request #12 from jrtechs/berkeley-transition

Migrated to using a Berkely backend

conf/graph.properties

@@ -0,0 +1,6 @@
+gremlin.graph=org.janusgraph.core.JanusGraphFactory
+
+storage.backend=berkeleyje
+
+storage.directory=./steam-graph/db
+query.smart-limit=false

src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java

@@ -128,6 +128,7 @@ public class APIConnection
             return friendsId; //private url
 
         JSONObject object = new JSONObject(apiData);
+        System.out.println(object);
 
         if(object.has("friendslist"))
         {

src/main/java/net/jrtechs/www/graphDB/GraphConnection.java

@@ -0,0 +1,40 @@
+package net.jrtechs.www.graphDB;
+
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.janusgraph.core.JanusGraphFactory;
+
+/**
+ * @author Jeffery Russell 7-10-2020
+ */
+public class GraphConnection
+{
+    private Graph graph;
+
+    private GraphTraversalSource t;
+
+    public GraphConnection()
+    {
+        this.graph = JanusGraphFactory.open("conf/graph.properties");
+        t = graph.traversal();
+    }
+
+
+    public void closeConnection() throws Exception
+    {
+        this.commit();
+        this.graph.close();
+    }
+
+    public void commit()
+    {
+        this.t.tx().commit();
+    }
+
+
+    public GraphTraversalSource getTraversal()
+    {
+        return this.t;
+    }
+}

src/main/java/net/jrtechs/www/graphDB/GremlinConsole.java

@@ -1,6 +1,7 @@
 package net.jrtechs.www.graphDB;
 
-import org.apache.tinkerpop.gremlin.driver.ResultSet;
+
+import org.apache.tinkerpop.gremlin.process.remote.RemoteConnection;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -17,7 +18,7 @@ import java.util.stream.IntStream;
 public class GremlinConsole
 {
     /** Connection to graph server **/
-    private RemoteConnection connection;
+    private GraphConnection connection;
 
 
     /**
@@ -25,7 +26,7 @@ public class GremlinConsole
      */
     public GremlinConsole()
     {
-        this.connection = new RemoteConnection();
+        this.connection = new GraphConnection();
     }
 
 
@@ -34,7 +35,7 @@ public class GremlinConsole
      *
      * @return
      */
-    public RemoteConnection getConnection()
+    public GraphConnection getConnection()
     {
         return this.connection;
     }
@@ -68,15 +69,17 @@ public class GremlinConsole
                     System.exit(0);
                 }
 
-                ResultSet set = this.connection.queryGraph(input);
-                try
-                {
-                    set.forEach(System.out::println);
-                }
-                catch (Exception ex)
-                {
-                    ex.printStackTrace();
-                }
+
+
+//                ResultSet set = this.connection.queryGraph(input);
+//                try
+//                {
+//                    set.forEach(System.out::println);
+//                }
+//                catch (Exception ex)
+//                {
+//                    ex.printStackTrace();
+//                }
 
             }
 
@@ -113,17 +116,24 @@ public class GremlinConsole
      *
      * @param args
      */
-    public static void main(String args[])
+    public static void main(String args[]) throws Exception
     {
-        GremlinConsole console = new GremlinConsole();
-
-        //don't worry about this lambda
-        IntStream.range(0, args.length)
-                .forEach(i-> console.getConnection()
-                        .queryGraph(args[i])
-                        .forEach((System.out::println)));
-
-        console.run();
+//        GremlinConsole console = new GremlinConsole();
+//
+//        //don't worry about this lambda
+//        IntStream.range(0, args.length)
+//                .forEach(i-> console.getConnection()
+//                        .queryGraph(args[i])
+//                        .forEach((System.out::println)));
+//
+//        console.run();
+
+        GraphConnection con = new GraphConnection();
+
+        System.out.println(con.getTraversal().E().toList()
+            );
+        System.out.println(con.getTraversal().V().toList()
+        );
+        con.closeConnection();
     }
-
 }

src/main/java/net/jrtechs/www/graphDB/RemoteConnection.java

@@ -1,101 +1,102 @@
-package net.jrtechs.www.graphDB;
-
-import org.apache.tinkerpop.gremlin.driver.Client;
-import org.apache.tinkerpop.gremlin.driver.Cluster;
-import org.apache.tinkerpop.gremlin.driver.ResultSet;
-
-import net.jrtechs.www.utils.ConfigLoader;
-
-/**
- * Simple helper class which allows us to remotely connect to a
- * remote graph
- *
- * @author Jeffery Russell 5-24-18
- */
-public class RemoteConnection
-{
-
-    /** Stores/manages client connections **/
-    private Cluster cluster;
-
-
-    /**
-     * Connection to the graph db
-     */
-    private Client client;
-
-
-    /**
-     * Connects to a remote Graph database
-     *
-     * Check this link out to learn about Cluster.Builder()
-     * http://tinkerpop.apache.org/javadocs/3.3.3/core/org/apache/tinkerpop/
-     * gremlin/driver/Cluster.Builder.html
-     *
-     */
-    public RemoteConnection()
-    {
-        ConfigLoader conf = new ConfigLoader("GremlinServerConnection.json");
-
-        Cluster.Builder b = Cluster.build();
-        b.addContactPoint(conf.getValue("host"));
-        b.port(conf.getInt("port"));
-
-        b.credentials(conf.getValue("username"), conf.getValue("password"));
-
-        this.cluster = b.create();
-
-        this.client = cluster.connect();
-    }
-
-
-    /**
-     * Queries the graph and return the results which can be iterated over
-     *
-     * ex:
-     *  ResultSet results = remote.queryGraph("g.V().values('name')");
-     *
-     *  results.stream().forEach(result ->
-     *  {
-     *       String s = result.getString();
-     *       System.out.println("name: " + s);
-     *  });
-     *
-     * @param q
-     * @return
-     */
-    public ResultSet queryGraph(String q)
-    {
-        return this.client.submit(q);
-    }
-
-
-    /**
-     * Closes connection with remote database
-     */
-    public void closeConnection()
-    {
-        this.cluster.close();
-    }
-
-
-    /**
-     * testing method which will be removed soon
-     * @param args
-     */
-    public static void main(String args[])
-    {
-        RemoteConnection remote = new RemoteConnection();
-        ResultSet results = remote.queryGraph("g.V().values('name')");
-
-        //results.stream().forEach(System.out::println);
-
-        results.stream().forEach(result ->
-        {
-            String s = result.getString();
-            System.out.println("name: " + s);
-        });
-
-        remote.closeConnection();
-    }
-}
+//package net.jrtechs.www.graphDB;
+//
+//import org.apache.tinkerpop.gremlin.driver.Client;
+//import org.apache.tinkerpop.gremlin.driver.Cluster;
+//import org.apache.tinkerpop.gremlin.driver.ResultSet;
+//
+//import net.jrtechs.www.utils.ConfigLoader;
+//
+///**
+// * Simple helper class which allows us to remotely connect to a
+// * remote graph
+// *
+// * @author Jeffery Russell 5-24-18
+// */
+//public class RemoteConnection
+//{
+//
+//    /** Stores/manages client connections **/
+//    private Cluster cluster;
+//
+//
+//    /**
+//     * Connection to the graph db
+//     */
+//    private Client client;
+//
+//
+//    /**
+//     * Connects to a remote Graph database
+//     *
+//     * Check this link out to learn about Cluster.Builder()
+//     * http://tinkerpop.apache.org/javadocs/3.3.3/core/org/apache/tinkerpop/
+//     * gremlin/driver/Cluster.Builder.html
+//     *
+//     */
+//    public RemoteConnection()
+//    {
+//        ConfigLoader conf = new ConfigLoader("GremlinServerConnection.json");
+//
+//        Cluster.Builder b = Cluster.build();
+//        b.addContactPoint(conf.getValue("host"));
+//        b.port(conf.getInt("port"));
+//
+//        b.credentials(conf.getValue("username"), conf.getValue("password"));
+//
+//        this.cluster = b.create();
+//
+//        this.client = cluster.connect();
+//    }
+//
+//
+//    /**
+//     * Queries the graph and return the results which can be iterated over
+//     *
+//     * ex:
+//     *  ResultSet results = remote.queryGraph("g.V().values('name')");
+//     *
+//     *  results.stream().forEach(result ->
+//     *  {
+//     *       String s = result.getString();
+//     *       System.out.println("name: " + s);
+//     *  });
+//     *
+//     * @param q
+//     * @return
+//     */
+//    public ResultSet queryGraph(String q)
+//    {
+//        return this.client.submit(q);
+//    }
+//
+//
+//    /**
+//     * Closes connection with remote database
+//     */
+//    public void closeConnection()
+//    {
+//        this.cluster.close();
+//    }
+//
+//
+//    /**
+//     * testing method which will be removed soon
+//     * @param args
+//     */
+//    public static void main(String args[])
+//    {
+//
+//        RemoteConnection remote = new RemoteConnection();
+//        ResultSet results = remote.queryGraph("g.V().values('name')");
+//
+//        //results.stream().forEach(System.out::println);
+//
+//        results.stream().forEach(result ->
+//        {
+//            String s = result.getString();
+//            System.out.println("name: " + s);
+//        });
+//
+//        remote.closeConnection();
+//    }
+//}

src/main/java/net/jrtechs/www/graphDB/SteamGraph.java

@@ -10,14 +10,14 @@ import java.util.Map;
 
 /**
  * Does graph based operations with {@link Player}
- * and {@link RemoteConnection}
+ * and
  *
  * @author Jeffery Russell 5-26-17
  */
 public class SteamGraph
 {
     /** Connection to the graph server */
-    private RemoteConnection con;
+    private GraphConnection con;
 
     /** Connection to steam api */
     private APIConnection api;
@@ -29,7 +29,7 @@ public class SteamGraph
      */
     public SteamGraph()
     {
-        this.con = new RemoteConnection();
+        this.con = new GraphConnection();
         this.api = new APIConnection();
     }
 
@@ -42,10 +42,11 @@ public class SteamGraph
      */
     private boolean alreadyInGraph(String id)
     {
-        String query = "g.V().hasLabel('player')" +
-                ".has('id', '" + id + "')";
-        //System.out.println(query);
-        return (1 <= con.queryGraph(query).stream().count());
+        System.out.println("Checking id:" + id);
+        return !con.getTraversal()
+                .V().hasLabel("player")
+                .has("id", id)
+                .toList().isEmpty();
     }
 
 
@@ -55,18 +56,18 @@ public class SteamGraph
      *
      * @param
      */
-    private void insertPlayerIntoGraph(String id, String name)
+    private void insertPlayerIntoGraph(String id, String name, boolean check)
     {
         try
         {
-            if(!this.alreadyInGraph(id))
+            if(!check || !this.alreadyInGraph(id))
             {
-                String queryInsertPlayer = "g.addV('player')" +
-                        ".property('name', '" + name + "')" +
-                        ".property('crawled', '0')" +
-                        ".property('id', '" + id + "')";
                 System.out.println("inserting " + name + " into graph");
-                this.con.queryGraph(queryInsertPlayer);
+                this.con.getTraversal()
+                        .addV("player")
+                        .property("name", name)
+                        .property("crawled", 0)
+                        .property("id", id).id().next();
             }
         }
         catch (Exception e)
@@ -88,12 +89,12 @@ public class SteamGraph
     {
         try
         {
-            String query = "g.V().hasLabel('player')" +
-                    ".has('id', '" + p1 + "')" +
-                    ".both()" +
-                    ".has('id', '" + p2 + "')";
-            //System.out.println(query);
-            return (1 <= con.queryGraph(query).stream().count());
+            return !this.con.getTraversal()
+                    .V().hasLabel("player")
+                    .has("id", p1)
+                    .both()
+                    .has("id", p2)
+                    .toList().isEmpty();
         }
         catch(Exception e)
         {
@@ -111,20 +112,22 @@ public class SteamGraph
      */
     private void insertEdgeIntoGraph(String p1, String p2)
     {
+
         try
         {
             if(!this.edgeAlreadyInGraph(p1, p2))
             {
-                String query = "g.V().hasLabel('player')" +
-                        ".has('id', '" + p1 + "')" +
-                        ".as('p1')" +
-                        "V().hasLabel('player')" +
-                        ".has('id', '" + p2 + "')" +
-                        ".as('p2')" +
-                        ".addE('friends')" +
-                        ".from('p1').to('p2')";
-                //System.out.println(query);
-                this.con.queryGraph(query);
+                System.out.println("Inserting edge: " + p1 + ":" + p2);
+                this.con.getTraversal()
+                        .V()
+                        .hasLabel("player")
+                        .has("id", p1)
+                        .as("p1")
+                        .V().hasLabel("player")
+                        .has("id", p2)
+                        .as("p2")
+                        .addE("friends")
+                        .from("p1").to("p2").id().next();
             }
         }
         catch (Exception e)
@@ -145,11 +148,11 @@ public class SteamGraph
     {
         try
         {
-            String query = "g.V().hasLabel('player')" +
-                    ".has('id', '" + id + "')" +
-                    ".has('crawled', '0')";
-
-            return (1 != con.queryGraph(query).stream().count());
+            return this.con.getTraversal()
+                    .V().hasLabel("player")
+                    .has("id", id)
+                    .has("crawled", 0)
+                    .toList().isEmpty();
         }
         catch(Exception e)
         {
@@ -163,11 +166,10 @@ public class SteamGraph
     {
         try
         {
-            String query = "g.V().hasLabel('player')" +
-                    ".has('id', '" + id + "')" +
-                    ".property('crawled', '1')";
-
-            this.con.queryGraph(query);
+            this.con.getTraversal().V()
+                    .hasLabel("player")
+                    .has("id", id)
+                    .property("crawled", 1).id().next();
         }
         catch (Exception e)
         {
@@ -184,11 +186,11 @@ public class SteamGraph
      */
     private String getNameFromGraph(String id)
     {
-        String query = "g.V().hasLabel('player')" +
-                ".has('id', '" + id + "')" +
-                ".values('name')";
-        return this.con.queryGraph(query).stream()
-                .findFirst().get().getObject().toString();
+        return this.con.getTraversal().V()
+                .hasLabel("player")
+                .has("id", id)
+                .values("name")
+                .toStream().findFirst().get().toString();
     }
 
 
@@ -200,19 +202,19 @@ public class SteamGraph
      */
     private List<Player> getFriendsFromGraph(String id)
     {
+        System.out.println("fetching friends from graph");
         List<Player> friends = new ArrayList<>();
-
-        String query = "g.V().hasLabel('player')" +
-                ".has('id', '" + id + "')" +
-                ".both().valueMap()";
         try
         {
-            this.con.queryGraph(query).stream().forEach(r ->
-                    friends.add(new Player(
-                            ((ArrayList) (((HashMap<String, Object>) (r.getObject()))
-                                    .get("name"))).get(0).toString(),
-                            ((ArrayList) (((HashMap<String, Object>) (r.getObject()))
-                                    .get("id"))).get(0).toString()))
+            this.con.getTraversal().V()
+                    .hasLabel("player")
+                    .has("id", id)
+                    .both().valueMap().toStream().forEach(r ->
+                        friends.add(
+                                new Player(r.get("name").toString(),
+                                        r.get("id").toString()
+                                )
+                        )
             );
         }
         catch(Exception e)
@@ -247,12 +249,13 @@ public class SteamGraph
 
         for(String key: names.keySet())
         {
-            this.insertPlayerIntoGraph(key, names.get(key));
+            this.insertPlayerIntoGraph(key, names.get(key), false);
         }
 
         friendsIds.forEach(s-> this.insertEdgeIntoGraph(id, s));
 
         this.updateCrawledStatus(id);
+        this.con.commit();
     }
 
 
@@ -285,13 +288,25 @@ public class SteamGraph
             }
             else
             {
-                this.insertPlayerIntoGraph(id, name);
+                this.insertPlayerIntoGraph(id, name, false);
                 return this.getPlayer(id);
             }
         }
         return p;
     }
 
+    public void close()
+    {
+        try
+        {
+            this.con.closeConnection();
+        }
+        catch (Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
 
     /**
      *
@@ -299,7 +314,10 @@ public class SteamGraph
      */
     public static void main(String[] args)
     {
-//        SteamGraph graph = new SteamGraph();
+        SteamGraph graph = new SteamGraph();
+        graph.getPlayer("76561198013779806").getFriends().stream().forEach(System.out::println);
+//        graph.indexPersonFriends("76561198188400721");
+        graph.close();
 //
 //        Player base = graph.getPlayer(args[0]);
 //

src/main/java/net/jrtechs/www/server/Player.java

@@ -38,15 +38,17 @@ public class Player
         this.id = id;
         this.friends = null;
         this.date = new Date();
-
+        this.friends = new ArrayList<>();
     }
 
 
-    public List<Player> getFriends() {
+    public List<Player> getFriends()
+    {
         return friends;
     }
 
-    public Date getDate() {
+    public Date getDate()
+    {
         return date;
     }
 
@@ -90,6 +92,6 @@ public class Player
     @Override
     public String toString()
     {
-        return "Name: " + this.name + " id: " + this.id;
+        return "Name: " + this.name + " id: " + this.id + " " + friends.size();
     }
 }