Browse Source

Created a class which lets me scrape data and insert it into the graph database.

pull/8/head
jrtechs 6 years ago
parent
commit
3dceab620e
4 changed files with 289 additions and 16 deletions
  1. +100
    -0
      src/main/java/net/jrtechs/www/Player.java
  2. +16
    -8
      src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java
  3. +155
    -0
      src/main/java/net/jrtechs/www/graphDB/SteamGraph.java
  4. +18
    -8
      src/main/java/net/jrtechs/www/utils/WebScraper.java

+ 100
- 0
src/main/java/net/jrtechs/www/Player.java View File

@ -0,0 +1,100 @@
package net.jrtechs.www;
import net.jrtechs.www.SteamAPI.APIConnection;
import java.util.ArrayList;
import java.util.List;
/**
* Class to store information on a player
*
* @author Jeffery Russell 5-26-18
*/
public class Player
{
/** Name of the player **/
private String name;
/** Steam id of the player **/
private String id;
/** List of friends the player has */
private List<Player> friends;
/**
* Sets the name and id of the player
*
* @param name
* @param id
*/
public Player(String name, String id)
{
this.name = name;
this.id = id;
this.friends = null;
}
/**
* Sets the fields of the player only based on it's
* steam id
*
* @param id
*/
public Player(String id)
{
this.id = id;
this.friends = null;
this.name = new APIConnection().getPlayerName(id);
}
/**
* Returns a list of all the friends of a specific player
*
* @param con
* @return
*/
public List<Player> fetchFriends(APIConnection con)
{
if(this.friends == null)
{
this.friends = new ArrayList<>();
con.getFriends(this.id)
.forEach(f-> this.friends.add(
new Player(con.getPlayerName(f), f)));
}
return friends;
}
/**
* Getter for display name of player
*
* @return
*/
public String getName()
{
return this.name.replace("'", "");
}
/**
* Getter for id of player
*
* @return
*/
public String getId()
{
return this.id;
}
@Override
public String toString()
{
return "Name: " + this.name + " id: " + this.id;
}
}

+ 16
- 8
src/main/java/net/jrtechs/www/SteamAPI/APIConnection.java View File

@ -53,14 +53,22 @@ public class APIConnection
{ {
List<String> friendsId = new ArrayList<>(); List<String> friendsId = new ArrayList<>();
new JSONObject(WebScraper
.getWebsite(this.baseURL + this.friendListURL +
this.apiKey + "&steamid=" + steamid))
.getJSONObject("friendslist")
.getJSONArray("friends").toList()
.forEach(f->
friendsId.add(((HashMap<String, String>)(f)).get("steamid"))
);
try
{
new JSONObject(WebScraper
.getWebsite(this.baseURL + this.friendListURL +
this.apiKey + "&steamid=" + steamid))
.getJSONObject("friendslist")
.getJSONArray("friends").toList()
.forEach(f->
friendsId.add(((HashMap<String, String>)(f)).get("steamid"))
);
}
catch (Exception ex)
{
ex.printStackTrace();
}
return friendsId; return friendsId;
} }

+ 155
- 0
src/main/java/net/jrtechs/www/graphDB/SteamGraph.java View File

@ -0,0 +1,155 @@
package net.jrtechs.www.graphDB;
import net.jrtechs.www.Player;
import net.jrtechs.www.SteamAPI.APIConnection;
/**
* Does graph based operations with {@link Player}
* and {@link RemoteConnection}
*
* @author Jeffery Russell 5-26-17
*/
public class SteamGraph
{
/** Connection to the graph server */
private RemoteConnection con;
/** Connection to steam api */
private APIConnection api;
/**
* Constructs object with a graph connection
* and a steam api connection
*/
public SteamGraph()
{
this.con = new RemoteConnection();
this.api = new APIConnection();
}
/**
* Checks if a player is already in the graph
*
* @param id steam id of player
* @return
*/
private boolean alreadyInGraph(String id)
{
String query = "g.V().hasLabel('player')" +
".has('id', '" + id + "')";
System.out.println(query);
return (1 == con.queryGraph(query).stream().count());
}
/**
* Checks if a friend-friend edge is already in the
* graph
*
* @param p1
* @param p2
* @return
*/
private boolean edgeAlreadyInGraph(Player p1, Player p2)
{
String query = "g.V().hasLabel('player')" +
".has('id', '" + p1.getId() + "')" +
".both()" +
".has('id', '" + p2.getId() + "')";
System.out.println(query);
return (1 == con.queryGraph(query).stream().count());
}
/**
* Inserts a player vertex into the graph
*
* @param player
*/
private void insertSinglePlayer(Player player)
{
String queryInsertPlayer = "g.addV('player')" +
".property('name', '" + player.getName() + "')" +
".property('id', '" + player.getId() + "')";
System.out.println(queryInsertPlayer);
this.con.queryGraph(queryInsertPlayer);
}
/**
* Inserts a edge between two players into the graph
*
* @param p1
* @param p2
*/
private void insertEdge(Player p1, Player p2)
{
String query = "g.V().hasLabel('player')" +
".has('id', '" + p1.getId() + "')" +
".as('p1')" +
"V().hasLabel('player')" +
".has('id', '" + p2.getId() + "')" +
".as('p2')" +
".addE('friends')" +
".from('p1').to('p2')";
System.out.println(query);
this.con.queryGraph(query);
}
/**
* Inserts a player and all of it's friends into
* the graph.
*
* @param player
*/
public void insertIntoGraph(Player player)
{
System.out.println(player);
if(!this.alreadyInGraph(player.getId()))
{
this.insertSinglePlayer(player);
}
for(Player friend : player.fetchFriends(api))
{
if(!alreadyInGraph(friend.getId()))
{
insertSinglePlayer(friend);
}
if(!edgeAlreadyInGraph(player, friend))
{
insertEdge(player, friend);
}
}
}
public void insertIntoGraph(Player player, int debth)
{
insertIntoGraph(player);
if(debth > 0)
{
player.fetchFriends(this.api)
.forEach(f -> insertIntoGraph(f, debth -1));
}
}
/**
*
* @param args
*/
public static void main(String[] args)
{
SteamGraph graph = new SteamGraph();
Player base = new Player(args[0]);
int debth = Integer.valueOf(args[1]);
graph.insertIntoGraph(base, debth);
}
}

+ 18
- 8
src/main/java/net/jrtechs/www/utils/WebScraper.java View File

@ -9,22 +9,27 @@ import java.net.URL;
* *
* @author Jeffery Russell 5-25-18 * @author Jeffery Russell 5-25-18
*/ */
public class WebScraper {
public class WebScraper
{
/** /**
* Grabs the contents of a website as a string * Grabs the contents of a website as a string
* *
* @param link to open * @param link to open
* @return source code of website as a single string * @return source code of website as a single string
*/ */
public static String getWebsite(String link) {
try {
public static String getWebsite(String link)
{
try
{
URL url = new URL(link); URL url = new URL(link);
BufferedReader br = new BufferedReader( BufferedReader br = new BufferedReader(
new InputStreamReader(url.openStream()) new InputStreamReader(url.openStream())
); );
return WebScraper.getBufferedReaderData(br); return WebScraper.getBufferedReaderData(br);
} catch (Exception e) {
}
catch (Exception e)
{
e.printStackTrace(); e.printStackTrace();
} }
return ""; return "";
@ -38,8 +43,10 @@ public class WebScraper {
* @param postData to send * @param postData to send
* @return source code of website as a string * @return source code of website as a string
*/ */
public static String getPostResponse(String link, String postData) {
try {
public static String getPostResponse(String link, String postData)
{
try
{
URL url = new URL(link); URL url = new URL(link);
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setDoOutput(true); conn.setDoOutput(true);
@ -53,7 +60,9 @@ public class WebScraper {
new InputStreamReader(conn.getInputStream()) new InputStreamReader(conn.getInputStream())
); );
return WebScraper.getBufferedReaderData(br); return WebScraper.getBufferedReaderData(br);
} catch (Exception ex) {
}
catch (Exception ex)
{
ex.printStackTrace(); ex.printStackTrace();
} }
@ -73,7 +82,8 @@ public class WebScraper {
throws IOException { throws IOException {
String html = ""; String html = "";
String line; String line;
while ((line = br.readLine()) != null) {
while ((line = br.readLine()) != null)
{
html += line; html += line;
} }
return html; return html;

Loading…
Cancel
Save