From c6be51f15fac458fc6821c90246913c1af4026de Mon Sep 17 00:00:00 2001 From: jrtechs Date: Thu, 26 Dec 2019 21:36:27 -0500 Subject: [PATCH 1/4] Added route in backend to fetch all friends(followers and following). --- routes/api.js | 150 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 22 deletions(-) diff --git a/routes/api.js b/routes/api.js index 6fba983..7b3d667 100644 --- a/routes/api.js +++ b/routes/api.js @@ -6,50 +6,155 @@ const GITHUB_API = "https://api.github.com"; const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`; +function queryGithubAPIRaw(requestURL) +{ + return new Promise((resolve, reject)=> + { + var queryURL; + if(requestURL.includes("?page=")) + { + queryURL = GITHUB_API + requestURL + "&" + authenticate; + } + else + { + queryURL = GITHUB_API + requestURL + "?" + authenticate; + } + console.log(queryURL); + + got(queryURL, { json: true }).then(response => + { + resolve(response.body); + cache.put(requestURL, response.body); + }).catch(error => + { + resolve(error); + cache.put(requestURL, error); + }); + }); +} + function queryGitHubAPI(requestURL) { const apiData = cache.get(requestURL); - return new Promise(function(reject, resolve) + return new Promise(function(resolve, reject) { if(apiData == null) { - var queryURL; - if(requestURL.includes("?page=")) + queryGithubAPIRaw(requestURL).then((dd)=> + { + resolve(dd); + }).catch((err)=> { - queryURL = GITHUB_API + requestURL + "&" + authenticate; + resolve(err); + }) + } + else + { + console.log("Fetched From Cache"); + resolve(apiData); + } + }) +} + + + +const API_FOLLOWING = "/following"; +const API_FOLLOWERS = "/followers"; +const API_USER_PATH = "/users/"; + +function fetchAllUsers(username, apiPath, page, lst) +{ + return new Promise((resolve, reject)=> + { + queryGitHubAPI(API_USER_PATH + username + apiPath + "?page=" + page).then((data)=> + { + if(data.hasOwnProperty("length")) + { + lst = lst.concat(data) + if(page < 5 && data.length === 30) + { + fetchAllUsers(username, apiPath, page + 1, lst).then((l)=> + { + resolve(l); + }); + } + else + { + resolve(lst); + } } else { - queryURL = GITHUB_API + requestURL + "?" + authenticate; + reject("Malformed data"); } - console.log(queryURL); + }).catch((err)=> + { + reject("error with api request"); + }); + }, + (error)=> + { + if(error.hasOwnProperty("length")) + { + lst.concat(data); + resolve(lst); + } + }); +} - got(queryURL, { json: true }).then(response => +function queryFriends(user) +{ + return new Promise((resolve, reject)=> + { + fetchAllUsers(user, API_FOLLOWERS, 1, []).then((followers)=> + { + fetchAllUsers(user, API_FOLLOWING, 1, []).then((following)=> { - resolve(response.body); - cache.put(requestURL, response.body); - }).catch(error => + resolve(following.concat(followers)); + }).catch((err)=> { - resolve(error); - cache.put(requestURL, error); - }); - } - else + console.log(err); + }) + }).catch((error)=> { - console.log("Fetched From Cahce"); - resolve(apiData); - } - }) + console.log(error); + }) + }); } +routes.get("/friends", (request, result)=> +{ + if(request.query.hasOwnProperty("name")) + { + result.setHeader('Content-Type', 'application/json'); + queryFriends(request.query.name).then(friends=> + { + result.write(JSON.stringify(friends)); + result.end(); + }).catch(error=> + { + result.status(500); + result.write("API error fetching friends.") + result.end(); + }); + } + else + { + result.status(400); + result.write("Must provide the name get parameter"); + result.end(); + } +}) + + routes.get('/*', (request, result) => { var gitHubAPIURL = request.url; result.setHeader('Content-Type', 'application/json'); - queryGitHubAPI(gitHubAPIURL).then(function(data) + queryGitHubAPI(gitHubAPIURL).then((data)=> { if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id")) { @@ -60,7 +165,7 @@ routes.get('/*', (request, result) => result.write("[]"); } result.end(); - }).catch(function(error) + }).catch((error)=> { try { @@ -74,7 +179,8 @@ routes.get('/*', (request, result) => } } - catch(error) { + catch(error) + { result.write("[]"); }; result.end(); From 8aac54d458e43c4528e345a6ebd9b6e0609b68fd Mon Sep 17 00:00:00 2001 From: jrtechs Date: Fri, 27 Dec 2019 13:49:27 -0500 Subject: [PATCH 2/4] Improved pagination by setting return size to 100 rather than default 20. Improved friend api structure. --- routes/api.js | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/routes/api.js b/routes/api.js index 7b3d667..c7c7158 100644 --- a/routes/api.js +++ b/routes/api.js @@ -62,17 +62,20 @@ function queryGitHubAPI(requestURL) const API_FOLLOWING = "/following"; const API_FOLLOWERS = "/followers"; const API_USER_PATH = "/users/"; +const API_PAGINATION_SIZE = 100; +const API_MAX_PAGES = 3; +const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE; function fetchAllUsers(username, apiPath, page, lst) { return new Promise((resolve, reject)=> { - queryGitHubAPI(API_USER_PATH + username + apiPath + "?page=" + page).then((data)=> + queryGitHubAPI(API_USER_PATH + username + apiPath + "?page=" + page + API_PAGINATION).then((data)=> { if(data.hasOwnProperty("length")) { lst = lst.concat(data) - if(page < 5 && data.length === 30) + if(page < API_MAX_PAGES && data.length === API_PAGINATION_SIZE) { fetchAllUsers(username, apiPath, page + 1, lst).then((l)=> { @@ -124,28 +127,18 @@ function queryFriends(user) } -routes.get("/friends", (request, result)=> +routes.get("/friends/:name", (request, result)=> { - if(request.query.hasOwnProperty("name")) + queryFriends(request.params.name).then(friends=> { - result.setHeader('Content-Type', 'application/json'); - queryFriends(request.query.name).then(friends=> - { - result.write(JSON.stringify(friends)); - result.end(); - }).catch(error=> - { - result.status(500); - result.write("API error fetching friends.") - result.end(); - }); - } - else + result.json(friends) + .end(); + }).catch(error=> { - result.status(400); - result.write("Must provide the name get parameter"); - result.end(); - } + result.status(500) + .json({error: 'API error fetching friends'}) + .end(); + }); }) From 1aac397daa3acfcd773ce98a51d3a549418218db Mon Sep 17 00:00:00 2001 From: jrtechs Date: Fri, 27 Dec 2019 15:10:42 -0500 Subject: [PATCH 3/4] Updated the front end to use the new friends backend api route --- public/js/friendsGraph.js | 123 ++++++++++++-------------------------- public/js/githubAPI.js | 81 +++++++++++++++++-------- 2 files changed, 93 insertions(+), 111 deletions(-) diff --git a/public/js/friendsGraph.js b/public/js/friendsGraph.js index f3baa11..794d4fb 100644 --- a/public/js/friendsGraph.js +++ b/public/js/friendsGraph.js @@ -75,12 +75,12 @@ function addPersonToGraph(profileData) * @param apiPath * @returns {Promise} */ -function addFriends(username, apiPath, page) +function addFriends(username) { updateProgress(); - return new Promise(function(resolve, reject) + return new Promise((resolve, reject)=> { - queryAPIByUser(apiPath + "?page=" + page, username, function(data) + getFriendsAPI(username, (data)=> { for(var i = 0; i < data.length; i++) { @@ -89,20 +89,9 @@ function addFriends(username, apiPath, page) addPersonToGraph(data[i]); } } - - if(page < 50 && data.length === 30) - { - addFriends(username, apiPath, page+ 1).then(function() - { - resolve(); - }) - } - else - { - resolve(); - } + resolve(); }, - function(error) + (error)=> { reject(error); }) @@ -156,37 +145,6 @@ function addConnection(person1, person2) } -function processConnections(user, apiPoint, page) -{ - return new Promise(function(resolve, reject) - { - queryAPIByUser(apiPoint + "?page=" + page, user.name, - function(data) - { - for(var i = 0; i < data.length; i++) - { - addConnection(user, data[i]) - } - if(page < 50 && data.length === 30) - { - processConnections(user, apiPoint, page + 1).then(function() - { - resolve(); - }); - } - else - { - resolve(); - } - }, function(error) - { - console.log(error); - resolve(); - }) - }) -} - - /** * Processes all the connections of a user and adds them to the graph * @@ -197,25 +155,20 @@ function processUserConnections(user) { return new Promise(function(resolve, reject) { - if(user.id === baseID) - { - processConnections(user, API_FOLLOWING, 1).then(function() + updateProgress(); + getFriendsAPI(user.name, + (data)=> { - processConnections(user, API_FOLLOWERS, 1).then(function() + for(var i = 0; i < data.length; i++) { - updateProgress(); - resolve(); - }) - }) - } - else - { - processConnections(user, API_FOLLOWING, 1).then(function() + addConnection(user, data[i]) + } + resolve(); + }, (error)=> { - updateProgress(); + console.log(error); resolve(); }) - } }); } @@ -228,7 +181,7 @@ function processUserConnections(user) */ function createConnections() { - return new Promise(function(resolve, reject) + return new Promise((resolve, reject)=> { var prom = []; for(var i = 0; i < nodes.length; i++) @@ -248,6 +201,9 @@ function createConnections() } +/** + * Updates progress bar for loading the JS graph + */ function updateProgress() { indexed++; @@ -267,16 +223,16 @@ function updateProgress() */ function addSelfToGraph(username) { - return new Promise(function(resolve, reject) + return new Promise((resolve, reject)=> { - queryAPIByUser("", username, function(data) + queryAPIByUser("", username, (data)=> { baseID = data.id; total = (data.followers + data.following); addPersonToGraph(data); resolve(); }, - function(error) + (error)=> { reject(error); }); @@ -313,35 +269,32 @@ function createFriendsGraph(username, containerName, progressBarID) nodes = []; edges = []; - addSelfToGraph(username).then(function() + addSelfToGraph(username).then(()=> { - addFriends(username, API_FOLLOWERS,1).then(function() + addFriends(username).then(()=> { - addFriends(username, API_FOLLOWING,1).then(function() + createConnections().then(()=> { - createConnections().then(function() - { - $("#" + progressID).html(""); + $("#" + progressID).html(""); - var container = document.getElementById(containerName); - var data = - { - nodes: nodes, - edges: edges - }; - var network = new vis.Network(container, data, options); + var container = document.getElementById(containerName); + var data = + { + nodes: nodes, + edges: edges + }; + var network = new vis.Network(container, data, options); - network.on("click", function (params) + network.on("click", function (params) + { + if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN) { - if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN) - { - bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM))); - } - }); + bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM))); + } }); }); }) - }).catch(function(error) + }).catch((error)=> { $("#" + graphsTitle).html("Error Fetching Data From API"); alert("Invalid User"); diff --git a/public/js/githubAPI.js b/public/js/githubAPI.js index 0dbe0a6..beb8030 100644 --- a/public/js/githubAPI.js +++ b/public/js/githubAPI.js @@ -17,10 +17,6 @@ const API_ORG_MEMBERS = "/members"; const API_REPOS = "/repos"; -const API_FOLLOWING = "/following"; - -const API_FOLLOWERS = "/followers"; - const API_REPOSITORIES = "/repos"; const API_ORGANIZATIONS = "/orgs"; @@ -31,39 +27,72 @@ const API_ORGANIZATIONS = "/orgs"; * allows you to get at the data using a * callback which gives you a json object. * - * @param apiPath the path on the github api ie API_FOLLOWING * @param user the username to query * @param successCallBack callback to complete when data is returned * @param errorCallBack callback which is invoked on error */ -function queryAPIByUser(apiPath, user, successCallBack, errorCallBack) { +function queryAPIByUser(apiPath, user, successCallBack, errorCallBack) +{ const urlpath = APIROOT + API_USER_PATH + user + apiPath; - console.log(urlpath); - $.ajax({ - type:'GET', - url: urlpath, - dataType: "json", - success: successCallBack, - error:errorCallBack, - timeout: 4000 - }); + runAjax(urlpath, successCallBack, errorCallBack); } -function queryAPIByOrg(apiPath, org, successCallBack, errorCallBack) { +/** + * Makes API calls for orgs on github + * + * @param {*} apiPath + * @param {*} org + * @param {*} successCallBack + * @param {*} errorCallBack + */ +function queryAPIByOrg(apiPath, org, successCallBack, errorCallBack) +{ const urlpath = APIROOT + API_ORG_PATH + org + apiPath; - console.log(urlpath); - $.ajax({ - type:'GET', - url: urlpath, - dataType: "json", - success: successCallBack, - error:errorCallBack, - timeout: 4000 - }); + runAjax(urlpath, successCallBack, errorCallBack); } -function queryUrl(url, successCallBack, errorCallBack) { + +/** + * Fetches a list of fiends for a user. + * + * @param {*} userName + * @param {*} suc + * @param {*} err + */ +function getFriendsAPI(userName, suc, err) +{ + // api/friends/jrtechs + const urlpath = APIROOT + "/friends/" + userName; + runAjax(urlpath, suc, err); +} + + +/** + * Queries github API end points with the backend + * proxy server for github graphs. + * + * @param {*} url + * @param {*} successCallBack + * @param {*} errorCallBack + */ +function queryUrl(url, successCallBack, errorCallBack) +{ url = url.split("https://api.github.com/").join("api/"); + runAjax(url, successCallBack, errorCallBack); +} + + +/** + * Wrapper for AJAX calls so we can unify + * all of our settings. + * + * @param {*} url -- url to query + * @param {*} successCallBack -- callback with data retrieved + * @param {*} errorCallBack -- callback with error message + */ +function runAjax(url, successCallBack, errorCallBack) +{ + console.log(url); $.ajax({ type:'GET', url: url, From b84536b5bcf306290accda7751d10e5b1527febd Mon Sep 17 00:00:00 2001 From: jrtechs Date: Fri, 27 Dec 2019 20:40:59 -0500 Subject: [PATCH 4/4] Minimized the json being stored and transmitted for the friends. --- routes/api.js | 100 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 11 deletions(-) diff --git a/routes/api.js b/routes/api.js index c7c7158..fe23ef8 100644 --- a/routes/api.js +++ b/routes/api.js @@ -6,6 +6,14 @@ const GITHUB_API = "https://api.github.com"; const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`; +/** + * Queries data from the github APi server and returns it as + * a json object in a promise. + * + * This makes no attempt to cache + * + * @param {*} requestURL endpoint on githubapi: ex: /users/jrtechs/following + */ function queryGithubAPIRaw(requestURL) { return new Promise((resolve, reject)=> @@ -33,6 +41,13 @@ function queryGithubAPIRaw(requestURL) }); } + +/** + * Queries data from the github api server + * and caches the results locally. + * + * @param {*} requestURL + */ function queryGitHubAPI(requestURL) { const apiData = cache.get(requestURL); @@ -58,19 +73,33 @@ function queryGitHubAPI(requestURL) } - const API_FOLLOWING = "/following"; const API_FOLLOWERS = "/followers"; const API_USER_PATH = "/users/"; -const API_PAGINATION_SIZE = 100; +const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default +// if this is too large, it would be infeasible to make graphs for people following popular people const API_MAX_PAGES = 3; const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE; + +/** + * This will fetch all of the either followers or following of a + * github user. + * + * This function is recursive to traverse all the pagination so we + * can get a complete list of all the friends. The max amount of + * followers/following you can get at once is 100. + * + * @param {*} username username of github client + * @param {*} apiPath following or followers + * @param {*} page current pagination page + * @param {*} lst list we are building on + */ function fetchAllUsers(username, apiPath, page, lst) { return new Promise((resolve, reject)=> { - queryGitHubAPI(API_USER_PATH + username + apiPath + "?page=" + page + API_PAGINATION).then((data)=> + queryGithubAPIRaw(API_USER_PATH + username + apiPath + "?page=" + page + API_PAGINATION).then((data)=> { if(data.hasOwnProperty("length")) { @@ -106,23 +135,72 @@ function fetchAllUsers(username, apiPath, page, lst) }); } + +/** + * Combines the list of friends and followers ignoring duplicates + * that are already in the list. (person is both following and followed by someone) + * + * This also removes any unused properties like events_url and organizations_url + * + * @param {*} followingAndFollowers + */ +function minimizeFriends(people) +{ + var friendLst = []; + + var ids = new Set(); + + for(var i = 0; i < people.length; i++) + { + if(!ids.has(people[i].id)) + { + ids.add(people[i].id); + friendLst.push({ + id: people[i].id, + login: people[i].login, + avatar_url: people[i].avatar_url + }); + } + } + return friendLst; +} + + +/** + * Fetches all the people that are either following or is followed + * by a person on github. This will cache the results to make simultaneous + * connections easier and less demanding on the github API. + * + * @param {*} user + */ function queryFriends(user) { + const cacheHit = cache.get("/friends/" + user); return new Promise((resolve, reject)=> { - fetchAllUsers(user, API_FOLLOWERS, 1, []).then((followers)=> + if(cacheHit == null) { - fetchAllUsers(user, API_FOLLOWING, 1, []).then((following)=> + fetchAllUsers(user, API_FOLLOWERS, 1, []).then((followers)=> { - resolve(following.concat(followers)); - }).catch((err)=> + fetchAllUsers(user, API_FOLLOWING, 1, []).then((following)=> + { + var fList = minimizeFriends(following.concat(followers)); + resolve(fList); + cache.put("/friends/" + user, fList); + }).catch((err)=> + { + console.log(err); + }) + }).catch((error)=> { - console.log(err); + console.log(error); }) - }).catch((error)=> + } + else { - console.log(error); - }) + console.log("Friends cache hit"); + resolve(cacheHit); + } }); }