Browse Source

Merge pull request #25 from jrtechs/friends-graph-improvement

Friends Graph Efficiency Improvement (Closes #20)
performance-improvements
Jeffery Russell 4 years ago
committed by GitHub
parent
commit
a50c999e4d
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 289 additions and 130 deletions
  1. +38
    -85
      public/js/friendsGraph.js
  2. +55
    -26
      public/js/githubAPI.js
  3. +196
    -19
      routes/api.js

+ 38
- 85
public/js/friendsGraph.js View File

@ -75,12 +75,12 @@ function addPersonToGraph(profileData)
* @param apiPath * @param apiPath
* @returns {Promise<any>} * @returns {Promise<any>}
*/ */
function addFriends(username, apiPath, page)
function addFriends(username)
{ {
updateProgress(); updateProgress();
return new Promise(function(resolve, reject)
return new Promise((resolve, reject)=>
{ {
queryAPIByUser(apiPath + "?page=" + page, username, function(data)
getFriendsAPI(username, (data)=>
{ {
for(var i = 0; i < data.length; i++) for(var i = 0; i < data.length; i++)
{ {
@ -89,20 +89,9 @@ function addFriends(username, apiPath, page)
addPersonToGraph(data[i]); addPersonToGraph(data[i]);
} }
} }
if(page < 50 && data.length === 30)
{
addFriends(username, apiPath, page+ 1).then(function()
{
resolve();
})
}
else
{
resolve();
}
resolve();
}, },
function(error)
(error)=>
{ {
reject(error); reject(error);
}) })
@ -156,37 +145,6 @@ function addConnection(person1, person2)
} }
function processConnections(user, apiPoint, page)
{
return new Promise(function(resolve, reject)
{
queryAPIByUser(apiPoint + "?page=" + page, user.name,
function(data)
{
for(var i = 0; i < data.length; i++)
{
addConnection(user, data[i])
}
if(page < 50 && data.length === 30)
{
processConnections(user, apiPoint, page + 1).then(function()
{
resolve();
});
}
else
{
resolve();
}
}, function(error)
{
console.log(error);
resolve();
})
})
}
/** /**
* Processes all the connections of a user and adds them to the graph * Processes all the connections of a user and adds them to the graph
* *
@ -197,25 +155,20 @@ function processUserConnections(user)
{ {
return new Promise(function(resolve, reject) return new Promise(function(resolve, reject)
{ {
if(user.id === baseID)
{
processConnections(user, API_FOLLOWING, 1).then(function()
updateProgress();
getFriendsAPI(user.name,
(data)=>
{ {
processConnections(user, API_FOLLOWERS, 1).then(function()
for(var i = 0; i < data.length; i++)
{ {
updateProgress();
resolve();
})
})
}
else
{
processConnections(user, API_FOLLOWING, 1).then(function()
addConnection(user, data[i])
}
resolve();
}, (error)=>
{ {
updateProgress();
console.log(error);
resolve(); resolve();
}) })
}
}); });
} }
@ -228,7 +181,7 @@ function processUserConnections(user)
*/ */
function createConnections() function createConnections()
{ {
return new Promise(function(resolve, reject)
return new Promise((resolve, reject)=>
{ {
var prom = []; var prom = [];
for(var i = 0; i < nodes.length; i++) for(var i = 0; i < nodes.length; i++)
@ -248,6 +201,9 @@ function createConnections()
} }
/**
* Updates progress bar for loading the JS graph
*/
function updateProgress() function updateProgress()
{ {
indexed++; indexed++;
@ -267,16 +223,16 @@ function updateProgress()
*/ */
function addSelfToGraph(username) function addSelfToGraph(username)
{ {
return new Promise(function(resolve, reject)
return new Promise((resolve, reject)=>
{ {
queryAPIByUser("", username, function(data)
queryAPIByUser("", username, (data)=>
{ {
baseID = data.id; baseID = data.id;
total = (data.followers + data.following); total = (data.followers + data.following);
addPersonToGraph(data); addPersonToGraph(data);
resolve(); resolve();
}, },
function(error)
(error)=>
{ {
reject(error); reject(error);
}); });
@ -313,35 +269,32 @@ function createFriendsGraph(username, containerName, progressBarID)
nodes = []; nodes = [];
edges = []; edges = [];
addSelfToGraph(username).then(function()
addSelfToGraph(username).then(()=>
{ {
addFriends(username, API_FOLLOWERS,1).then(function()
addFriends(username).then(()=>
{ {
addFriends(username, API_FOLLOWING,1).then(function()
createConnections().then(()=>
{ {
createConnections().then(function()
{
$("#" + progressID).html("");
$("#" + progressID).html("");
var container = document.getElementById(containerName);
var data =
{
nodes: nodes,
edges: edges
};
var network = new vis.Network(container, data, options);
var container = document.getElementById(containerName);
var data =
{
nodes: nodes,
edges: edges
};
var network = new vis.Network(container, data, options);
network.on("click", function (params)
network.on("click", function (params)
{
if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN)
{ {
if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN)
{
bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM)));
}
});
bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM)));
}
}); });
}); });
}) })
}).catch(function(error)
}).catch((error)=>
{ {
$("#" + graphsTitle).html("Error Fetching Data From API"); $("#" + graphsTitle).html("Error Fetching Data From API");
alert("Invalid User"); alert("Invalid User");

+ 55
- 26
public/js/githubAPI.js View File

@ -17,10 +17,6 @@ const API_ORG_MEMBERS = "/members";
const API_REPOS = "/repos"; const API_REPOS = "/repos";
const API_FOLLOWING = "/following";
const API_FOLLOWERS = "/followers";
const API_REPOSITORIES = "/repos"; const API_REPOSITORIES = "/repos";
const API_ORGANIZATIONS = "/orgs"; const API_ORGANIZATIONS = "/orgs";
@ -31,39 +27,72 @@ const API_ORGANIZATIONS = "/orgs";
* allows you to get at the data using a * allows you to get at the data using a
* callback which gives you a json object. * callback which gives you a json object.
* *
* @param apiPath the path on the github api ie API_FOLLOWING
* @param user the username to query * @param user the username to query
* @param successCallBack callback to complete when data is returned * @param successCallBack callback to complete when data is returned
* @param errorCallBack callback which is invoked on error * @param errorCallBack callback which is invoked on error
*/ */
function queryAPIByUser(apiPath, user, successCallBack, errorCallBack) {
function queryAPIByUser(apiPath, user, successCallBack, errorCallBack)
{
const urlpath = APIROOT + API_USER_PATH + user + apiPath; const urlpath = APIROOT + API_USER_PATH + user + apiPath;
console.log(urlpath);
$.ajax({
type:'GET',
url: urlpath,
dataType: "json",
success: successCallBack,
error:errorCallBack,
timeout: 4000
});
runAjax(urlpath, successCallBack, errorCallBack);
} }
function queryAPIByOrg(apiPath, org, successCallBack, errorCallBack) {
/**
* Makes API calls for orgs on github
*
* @param {*} apiPath
* @param {*} org
* @param {*} successCallBack
* @param {*} errorCallBack
*/
function queryAPIByOrg(apiPath, org, successCallBack, errorCallBack)
{
const urlpath = APIROOT + API_ORG_PATH + org + apiPath; const urlpath = APIROOT + API_ORG_PATH + org + apiPath;
console.log(urlpath);
$.ajax({
type:'GET',
url: urlpath,
dataType: "json",
success: successCallBack,
error:errorCallBack,
timeout: 4000
});
runAjax(urlpath, successCallBack, errorCallBack);
} }
function queryUrl(url, successCallBack, errorCallBack) {
/**
* Fetches a list of fiends for a user.
*
* @param {*} userName
* @param {*} suc
* @param {*} err
*/
function getFriendsAPI(userName, suc, err)
{
// api/friends/jrtechs
const urlpath = APIROOT + "/friends/" + userName;
runAjax(urlpath, suc, err);
}
/**
* Queries github API end points with the backend
* proxy server for github graphs.
*
* @param {*} url
* @param {*} successCallBack
* @param {*} errorCallBack
*/
function queryUrl(url, successCallBack, errorCallBack)
{
url = url.split("https://api.github.com/").join("api/"); url = url.split("https://api.github.com/").join("api/");
runAjax(url, successCallBack, errorCallBack);
}
/**
* Wrapper for AJAX calls so we can unify
* all of our settings.
*
* @param {*} url -- url to query
* @param {*} successCallBack -- callback with data retrieved
* @param {*} errorCallBack -- callback with error message
*/
function runAjax(url, successCallBack, errorCallBack)
{
console.log(url);
$.ajax({ $.ajax({
type:'GET', type:'GET',
url: url, url: url,

+ 196
- 19
routes/api.js View File

@ -6,50 +6,226 @@ const GITHUB_API = "https://api.github.com";
const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`; const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`;
/**
* Queries data from the github APi server and returns it as
* a json object in a promise.
*
* This makes no attempt to cache
*
* @param {*} requestURL endpoint on githubapi: ex: /users/jrtechs/following
*/
function queryGithubAPIRaw(requestURL)
{
return new Promise((resolve, reject)=>
{
var queryURL;
if(requestURL.includes("?page="))
{
queryURL = GITHUB_API + requestURL + "&" + authenticate;
}
else
{
queryURL = GITHUB_API + requestURL + "?" + authenticate;
}
console.log(queryURL);
got(queryURL, { json: true }).then(response =>
{
resolve(response.body);
cache.put(requestURL, response.body);
}).catch(error =>
{
resolve(error);
cache.put(requestURL, error);
});
});
}
/**
* Queries data from the github api server
* and caches the results locally.
*
* @param {*} requestURL
*/
function queryGitHubAPI(requestURL) function queryGitHubAPI(requestURL)
{ {
const apiData = cache.get(requestURL); const apiData = cache.get(requestURL);
return new Promise(function(reject, resolve)
return new Promise(function(resolve, reject)
{ {
if(apiData == null) if(apiData == null)
{ {
var queryURL;
if(requestURL.includes("?page="))
queryGithubAPIRaw(requestURL).then((dd)=>
{ {
queryURL = GITHUB_API + requestURL + "&" + authenticate;
resolve(dd);
}).catch((err)=>
{
resolve(err);
})
}
else
{
console.log("Fetched From Cache");
resolve(apiData);
}
})
}
const API_FOLLOWING = "/following";
const API_FOLLOWERS = "/followers";
const API_USER_PATH = "/users/";
const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
// if this is too large, it would be infeasible to make graphs for people following popular people
const API_MAX_PAGES = 3;
const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
/**
* This will fetch all of the either followers or following of a
* github user.
*
* This function is recursive to traverse all the pagination so we
* can get a complete list of all the friends. The max amount of
* followers/following you can get at once is 100.
*
* @param {*} username username of github client
* @param {*} apiPath following or followers
* @param {*} page current pagination page
* @param {*} lst list we are building on
*/
function fetchAllUsers(username, apiPath, page, lst)
{
return new Promise((resolve, reject)=>
{
queryGithubAPIRaw(API_USER_PATH + username + apiPath + "?page=" + page + API_PAGINATION).then((data)=>
{
if(data.hasOwnProperty("length"))
{
lst = lst.concat(data)
if(page < API_MAX_PAGES && data.length === API_PAGINATION_SIZE)
{
fetchAllUsers(username, apiPath, page + 1, lst).then((l)=>
{
resolve(l);
});
}
else
{
resolve(lst);
}
} }
else else
{ {
queryURL = GITHUB_API + requestURL + "?" + authenticate;
reject("Malformed data");
} }
console.log(queryURL);
}).catch((err)=>
{
reject("error with api request");
});
},
(error)=>
{
if(error.hasOwnProperty("length"))
{
lst.concat(data);
resolve(lst);
}
});
}
/**
* Combines the list of friends and followers ignoring duplicates
* that are already in the list. (person is both following and followed by someone)
*
* This also removes any unused properties like events_url and organizations_url
*
* @param {*} followingAndFollowers
*/
function minimizeFriends(people)
{
var friendLst = [];
var ids = new Set();
for(var i = 0; i < people.length; i++)
{
if(!ids.has(people[i].id))
{
ids.add(people[i].id);
friendLst.push({
id: people[i].id,
login: people[i].login,
avatar_url: people[i].avatar_url
});
}
}
return friendLst;
}
got(queryURL, { json: true }).then(response =>
/**
* Fetches all the people that are either following or is followed
* by a person on github. This will cache the results to make simultaneous
* connections easier and less demanding on the github API.
*
* @param {*} user
*/
function queryFriends(user)
{
const cacheHit = cache.get("/friends/" + user);
return new Promise((resolve, reject)=>
{
if(cacheHit == null)
{
fetchAllUsers(user, API_FOLLOWERS, 1, []).then((followers)=>
{ {
resolve(response.body);
cache.put(requestURL, response.body);
}).catch(error =>
fetchAllUsers(user, API_FOLLOWING, 1, []).then((following)=>
{
var fList = minimizeFriends(following.concat(followers));
resolve(fList);
cache.put("/friends/" + user, fList);
}).catch((err)=>
{
console.log(err);
})
}).catch((error)=>
{ {
resolve(error);
cache.put(requestURL, error);
});
console.log(error);
})
} }
else else
{ {
console.log("Fetched From Cahce");
resolve(apiData);
console.log("Friends cache hit");
resolve(cacheHit);
} }
})
});
} }
routes.get("/friends/:name", (request, result)=>
{
queryFriends(request.params.name).then(friends=>
{
result.json(friends)
.end();
}).catch(error=>
{
result.status(500)
.json({error: 'API error fetching friends'})
.end();
});
})
routes.get('/*', (request, result) => routes.get('/*', (request, result) =>
{ {
var gitHubAPIURL = request.url; var gitHubAPIURL = request.url;
result.setHeader('Content-Type', 'application/json'); result.setHeader('Content-Type', 'application/json');
queryGitHubAPI(gitHubAPIURL).then(function(data)
queryGitHubAPI(gitHubAPIURL).then((data)=>
{ {
if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id")) if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id"))
{ {
@ -60,7 +236,7 @@ routes.get('/*', (request, result) =>
result.write("[]"); result.write("[]");
} }
result.end(); result.end();
}).catch(function(error)
}).catch((error)=>
{ {
try try
{ {
@ -74,7 +250,8 @@ routes.get('/*', (request, result) =>
} }
} }
catch(error) {
catch(error)
{
result.write("[]"); result.write("[]");
}; };
result.end(); result.end();

Loading…
Cancel
Save