Merge pull request #16 from jrtechs/game_data

Index Game Data

pom.xml

@@ -29,25 +29,10 @@
       <scope>test</scope>
     </dependency>
 
-    <!-- https://mvnrepository.com/artifact/com.tinkerpop/gremlin-core -->
     <dependency>
-      <groupId>com.tinkerpop</groupId>
-      <artifactId>gremlin-core</artifactId>
-      <version>3.0.0.M7</version>
-    </dependency>
-
-    <!-- https://mvnrepository.com/artifact/org.apache.tinkerpop/gremlin-driver -->
-    <dependency>
-      <groupId>org.apache.tinkerpop</groupId>
-      <artifactId>gremlin-driver</artifactId>
-      <version>3.3.3</version>
-    </dependency>
-
-
-    <dependency>
-      <groupId>org.apache.tinkerpop</groupId>
-      <artifactId>tinkergraph-gremlin</artifactId>
-      <version>3.3.3</version>
+      <groupId>org.janusgraph</groupId>
+      <artifactId>janusgraph-berkeleyje</artifactId>
+      <version>0.3.1</version>
     </dependency>
 
     <!-- https://mvnrepository.com/artifact/org.json/json -->

src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java

@@ -1,5 +1,6 @@
 package net.jrtechs.www.SteamAPI;
 
+import net.jrtechs.www.server.Game;
 import net.jrtechs.www.server.Player;
 import net.jrtechs.www.utils.ConfigLoader;
 
@@ -8,26 +9,33 @@ import net.jrtechs.www.webCrawler.APIThrottler;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
+import javax.json.JsonArray;
+import javax.json.JsonObject;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.IntStream;
 
 /**
  * Class which is used to pull information from the Steam api
  *
+ * Documentation at https://developer.valvesoftware.com/wiki/Steam_Web_API
+ *
  * @author Jeffery Russell 5-26-18
  */
 public class APIConnection
 {
     /** Base url to use for all queries to steam's api  **/
-    private final String baseURL = "http://api.steampowered.com";
+    private final String baseURL = "https://community.steam-api.com";
 
     /** Path to use when getting info on a player from api **/
     private final String playerInfoURL = "/ISteamUser/GetPlayerSummaries/v0002/";
 
     private final String friendListURL = "/ISteamUser/GetFriendList/v0001/";
 
+    private final String gamesListURL = "/IPlayerService/GetOwnedGames/v0001/";
+
     /** Path to conf file(from within the conf folder) **/
     private final String confPath = "SteamAPIKey.json";
 
@@ -57,6 +65,7 @@ public class APIConnection
      */
     public String querySteamAPI(String url)
     {
+        System.out.println(url);
         boolean downloaded = false;
         String apiData = "";
         while(!downloaded)
@@ -111,6 +120,29 @@ public class APIConnection
     }
 
 
+    public List<Game> getGames(String steamID)
+    {
+        List<Game> games = new ArrayList<>();
+        String apiData = this.querySteamAPI(this.baseURL + this.gamesListURL +
+                this.apiKey + "&steamid=" + steamID +
+                "&include_appinfo=true&include_played_free_games=true");
+
+        if(apiData.isEmpty())
+            return games;
+
+        JSONObject object = new JSONObject(apiData);
+        System.out.println(object);
+
+        if(object.has("response"))
+        {
+            JSONArray gamesJ = object.getJSONObject("response").getJSONArray("games");
+            IntStream.range(0, gamesJ.length()).forEach(i ->
+                    games.add(new Game(gamesJ.getJSONObject(i))));
+        }
+        return games;
+    }
+
+
     /**
      * Returns a list of the UIDs of all the players friends
      *
@@ -259,6 +291,7 @@ public class APIConnection
         //steam id of jrtechs
         con.getFriends("76561198188400721").forEach(System.out::println);
 
-        System.out.println(con.getSingle("76561198188400721"));
+        //System.out.println(con.getSingle("76561198188400721"));
+        System.out.println(con.getGames("76561198188400721"));
     }
 }

src/main/java/net/jrtechs/www/graphDB/SteamGraph.java

@@ -1,13 +1,12 @@
 package net.jrtechs.www.graphDB;
 
 import net.jrtechs.www.SteamAPI.SteamConnectionException;
+import net.jrtechs.www.server.Game;
 import net.jrtechs.www.server.Player;
 import net.jrtechs.www.SteamAPI.APIConnection;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -21,6 +20,7 @@ public class SteamGraph
     public static String KEY_PLAYER = "player";
     public static String KEY_CRAWLED_STATUS = "crawled";
 
+    public static String KEY_CRAWLED_GAME_STATUS = "crawled_game";
 
     /** Connection to the graph server */
     private GraphConnection con;
@@ -73,6 +73,7 @@ public class SteamGraph
                         .addV(SteamGraph.KEY_PLAYER)
                         .property(Player.KEY_USERNAME, p.getName())
                         .property(SteamGraph.KEY_CRAWLED_STATUS, 0)
+                        .property(SteamGraph.KEY_CRAWLED_GAME_STATUS, 0)
                         .property(Player.KEY_STEAM_ID, p.getId())
                         .property(Player.KEY_AVATAR, p.getAvatar())
                         .property(Player.KEY_REAL_NAME, p.getRealName())
@@ -86,6 +87,44 @@ public class SteamGraph
         }
     }
 
+    private boolean gameAlreadyInGraph(Integer id)
+    {
+        return !con.getTraversal()
+                .V().hasLabel(Game.KEY_DB)
+                .has(Game.KEY_STEAM_GAME_ID, id)
+                .toList().isEmpty();
+    }
+
+    private void insertGameForPlayerToGraph(String id, Game g)
+    {
+        if(!gameAlreadyInGraph(g.getAppID()))// check if game is already in graph
+        {
+            //insert game into graph
+            this.con.getTraversal()
+                    .addV(Game.KEY_DB)
+                    .property(Game.KEY_STEAM_GAME_ID, g.getAppID())
+                    .property(Game.KEY_GAME_NAME, g.getName())
+                    .property(Game.KEY_GAME_ICON, g.getIcon())
+                    .property(Game.KEY_GAME_LOGO, g.getLogo())
+                    .id().next();
+        }
+
+        // insert connection from player to game
+
+        this.con.getTraversal()
+                .V()
+                .hasLabel(SteamGraph.KEY_PLAYER)
+                .has(Player.KEY_STEAM_ID, id)
+                .as("p")
+                .V().hasLabel(Game.KEY_DB)
+                .has(Game.KEY_STEAM_GAME_ID, g.getAppID())
+                .as("g")
+                .addE(Game.KEY_RELATIONSHIP)
+                .from("p").to("g")
+                .property(Game.KEY_PLAY_TIME, g.getTimePlayed())
+                .id().next();
+    }
+
 
     /**
      * Checks if a friend-friend edge is already in the
@@ -122,7 +161,6 @@ public class SteamGraph
      */
     private void insertEdgeIntoGraph(String p1, String p2)
     {
-
         try
         {
             if(!this.edgeAlreadyInGraph(p1, p2))
@@ -154,14 +192,31 @@ public class SteamGraph
      * @param id
      * @return
      */
-    private boolean playerAlreadyIndexed(String id)
+    private boolean playerFriendsAlreadyIndexed(String id)
+    {
+        return playerPropertyIndexed(id, SteamGraph.KEY_CRAWLED_STATUS);
+    }
+
+
+    private boolean playerGamesAlreadyIndexed(String id)
+    {
+        return playerPropertyIndexed(id, SteamGraph.KEY_CRAWLED_GAME_STATUS);
+    }
+
+    /**
+     * determines if a player has been indexed yet
+     *
+     * @param id
+     * @return
+     */
+    private boolean playerPropertyIndexed(String id, String key)
     {
         try
         {
             return this.con.getTraversal()
                     .V().hasLabel(SteamGraph.KEY_PLAYER)
                     .has(Player.KEY_STEAM_ID, id)
-                    .has(SteamGraph.KEY_CRAWLED_STATUS, 0)
+                    .has(key, 0)
                     .toList().isEmpty();
         }
         catch(Exception e)
@@ -172,14 +227,26 @@ public class SteamGraph
     }
 
 
-    private void updateCrawledStatus(String id)
+    private void updateCrawledStatusFriends(String id)
+    {
+        updateCrawledStatus(id, SteamGraph.KEY_CRAWLED_STATUS);
+    }
+
+
+    private void updateCrawledStatusGames(String id)
+    {
+        updateCrawledStatus(id, SteamGraph.KEY_CRAWLED_GAME_STATUS);
+    }
+
+
+    private void updateCrawledStatus(String id, String key)
     {
         try
         {
             this.con.getTraversal().V()
                     .hasLabel(SteamGraph.KEY_PLAYER)
                     .has(Player.KEY_STEAM_ID, id)
-                    .property(SteamGraph.KEY_CRAWLED_STATUS, 1).id().next();
+                    .property(key, 1).id().next();
         }
         catch (Exception e)
         {
@@ -218,11 +285,33 @@ public class SteamGraph
             con.getTraversal().V()
                 .hasLabel(SteamGraph.KEY_PLAYER)
                 .has(Player.KEY_STEAM_ID, id)
-                .both().valueMap().toStream().forEach(r ->
+                .outE()
+                .inV()
+                .hasLabel(SteamGraph.KEY_PLAYER)
+                .valueMap()
+                .toStream()
+                .forEach(r ->
                     add(new Player(r)));
         }};
     }
 
+    private List<Game> getPlayerGamesFromGraph(String id)
+    {
+        System.out.println("fetching games from graph");
+        return new ArrayList<Game>()
+        {{
+            con.getTraversal().V()
+                    .hasLabel(SteamGraph.KEY_PLAYER)
+                    .has(Player.KEY_STEAM_ID, id)
+                    .outE()
+                    .inV()
+                    .hasLabel(Game.KEY_DB)
+                    .valueMap()
+                    .toStream().forEach(r ->
+                        add(new Game(r)));
+        }};
+    }
+
 
     /**
      * tells api to get this dude's friends list
@@ -248,10 +337,27 @@ public class SteamGraph
         friendsIds.forEach(s->
                 this.insertEdgeIntoGraph(id, s));
 
-        this.updateCrawledStatus(id);
+        this.updateCrawledStatusFriends(id);
         this.con.commit();
     }
 
+    private List<Game> indexPersonsGames(String id)
+    {
+        System.out.println("indexing  games for " + id);
+        List<Game> games = this.api.getGames(id);
+        games.forEach(g -> insertGameForPlayerToGraph(id, g));
+        this.updateCrawledStatusGames(id);
+        this.con.commit();
+        return games;
+    }
+
+    public List<Game> getGameList(String id)
+    {
+        return this.playerGamesAlreadyIndexed(id) ?
+                this.getPlayerGamesFromGraph(id) :
+                this.indexPersonsGames(id);
+    }
+
 
     /**
      * Fetches a player from the graph with all of its friends
@@ -265,7 +371,7 @@ public class SteamGraph
         if(this.alreadyInGraph(id)) // yay
         {
             p = this.getPlayerFromGraph(id);
-            if(!this.playerAlreadyIndexed(id)) //must index the person
+            if(!this.playerFriendsAlreadyIndexed(id)) //must index the person
             {
                 this.indexPersonFriends(id);
             }
@@ -279,6 +385,8 @@ public class SteamGraph
             {
                 p = this.api.getSingle(id);
                 this.insertPlayerIntoGraph(p, false);
+                this.indexPersonFriends(id);
+                p.setFriends(this.getFriendsFromGraph(id));
             }
             catch (SteamConnectionException e)
             {
@@ -309,8 +417,14 @@ public class SteamGraph
     public static void main(String[] args)
     {
         SteamGraph graph = new SteamGraph();
-        graph.getPlayer("76561198068098265").getFriends().stream().forEach(System.out::println);
-//        graph.indexPersonFriends("76561198188400721");
+
+        graph.getPlayer("76561198188400721");
+        graph.getGameList("76561198188400721");
+//        //graph.getPlayer("76561198068098265").getFriends().stream().forEach(System.out::println);
+////        graph.indexPersonFriends("76561198188400721");
+//        graph.indexPersonsGames("76561198188400721");
+//        System.out.println(graph.getGameList("76561198188400721"));
+//        System.out.println(graph.getPlayer("76561198188400721"));
         graph.close();
 //
 //        Player base = graph.getPlayer(args[0]);

src/main/java/net/jrtechs/www/server/Game.java

@@ -0,0 +1,88 @@
+package net.jrtechs.www.server;
+
+
+import org.json.JSONObject;
+
+import javax.json.JsonObject;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Example URL: http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/?key=XXXXXXXXXXXXXXXXX&steamid=76561197960434622&format=json
+ */
+public class Game
+{
+    public static String KEY_DB = "game";
+    public static String KEY_STEAM_GAME_ID = "appid";
+    public static String KEY_GAME_NAME = "name";
+    public static String KEY_GAME_ICON = "img_icon_url";
+    public static String KEY_GAME_LOGO = "img_logo_url";
+
+    public static String KEY_RELATIONSHIP = "owns";
+
+    //other
+    public static String KEY_PLAY_TIME = "playtime_forever";
+
+    private Integer appID;
+    private String icon;
+    private String logo;
+    private String name;
+
+    private Integer timePlayed;
+
+    public Game(JSONObject g)
+    {
+        this.appID = g.getInt(Game.KEY_STEAM_GAME_ID);
+        this.name = g.getString(KEY_GAME_NAME);
+        this.icon = g.getString(KEY_GAME_ICON);
+        this.logo = g.getString(KEY_GAME_LOGO);
+        this.timePlayed = g.getInt(KEY_PLAY_TIME);
+    }
+
+    public Game(Map<String, Object> graph)
+    {
+        System.out.println(graph);
+        this.appID= (Integer)((List<Object>) graph.get(KEY_STEAM_GAME_ID)).get(0);
+        this.name = (String)((List<Object>) graph.get(KEY_GAME_NAME)).get(0);
+        this.icon = (String)((List<Object>) graph.get(KEY_GAME_ICON)).get(0);
+        this.logo = (String)((List<Object>) graph.get(KEY_GAME_LOGO)).get(0);
+        this.timePlayed = 0;
+    }
+
+    public Integer getAppID()
+    {
+        return appID;
+    }
+
+    public String getIcon()
+    {
+        return icon;
+    }
+
+    public String getLogo()
+    {
+        return logo;
+    }
+
+    public String getName()
+    {
+        return name;
+    }
+
+    public Integer getTimePlayed()
+    {
+        return timePlayed;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Game{" +
+                "appID=" + appID +
+                ", icon='" + icon + '\'' +
+                ", logo='" + logo + '\'' +
+                ", name='" + name + '\'' +
+                ", timePlayed=" + timePlayed +
+                '}';
+    }
+}

src/main/java/net/jrtechs/www/server/Player.java

@@ -36,6 +36,7 @@ public class Player
 
     private Integer timeCreated;
 
+    private List<Game> playerGames;
 
     /**
      * Sets the name and id of the player
@@ -53,6 +54,7 @@ public class Player
         this.timeCreated = timeCreated;
         this.avatar = avatar;
         this.friends = new ArrayList<>();
+        this.playerGames = new ArrayList<>();
     }
 
 
@@ -64,6 +66,7 @@ public class Player
         this.avatar = ((List<Object>) apiInfo.getOrDefault(Player.KEY_AVATAR, "")).get(0).toString();
         this.timeCreated = (Integer)((List<Object>)apiInfo.get(KEY_TIME_CREATED)).get(0);
         this.friends = new ArrayList<>();
+        this.playerGames = new ArrayList<>();
     }
 
     public List<Player> getFriends()
@@ -71,6 +74,11 @@ public class Player
         return friends;
     }
 
+    public List<Game> getGames()
+    {
+        return this.playerGames;
+    }
+
 
     /**
      * Getter for display name of player