Merge pull request #25 from jrtechs/friends-graph-improvement

Friends Graph Efficiency Improvement (Closes #20)

public/js/friendsGraph.js

@@ -75,12 +75,12 @@ function addPersonToGraph(profileData)
  * @param apiPath
  * @returns {Promise<any>}
  */
-function addFriends(username, apiPath, page)
+function addFriends(username)
 {
     updateProgress();
-    return new Promise(function(resolve, reject)
+    return new Promise((resolve, reject)=>
     {
-        queryAPIByUser(apiPath + "?page=" + page, username, function(data)
+        getFriendsAPI(username, (data)=>
         {
             for(var i = 0; i < data.length; i++)
             {
@@ -89,20 +89,9 @@ function addFriends(username, apiPath, page)
                     addPersonToGraph(data[i]);
                 }
             }
-
-            if(page < 50 && data.length === 30)
-            {
-                addFriends(username, apiPath, page+ 1).then(function()
-                {
-                    resolve();
-                })
-            }
-            else
-            {
-                resolve();
-            }
+            resolve();
         },
-        function(error)
+        (error)=>
         {
             reject(error);
         })
@@ -156,37 +145,6 @@ function addConnection(person1, person2)
 }
 
 
-function processConnections(user, apiPoint, page)
-{
-    return new Promise(function(resolve, reject)
-    {
-        queryAPIByUser(apiPoint + "?page=" + page, user.name,
-            function(data)
-            {
-                for(var i = 0; i < data.length; i++)
-                {
-                    addConnection(user, data[i])
-                }
-                if(page < 50 && data.length === 30)
-                {
-                    processConnections(user, apiPoint, page + 1).then(function()
-                    {
-                        resolve();
-                    });
-                }
-                else
-                {
-                    resolve();
-                }
-            }, function(error)
-            {
-                console.log(error);
-                resolve();
-            })
-    })
-}
-
-
 /**
  * Processes all the connections of a user and adds them to the graph
  *
@@ -197,25 +155,20 @@ function processUserConnections(user)
 {
     return new Promise(function(resolve, reject)
     {
-        if(user.id === baseID)
-        {
-            processConnections(user, API_FOLLOWING, 1).then(function()
+        updateProgress();
+        getFriendsAPI(user.name,
+            (data)=>
             {
-                processConnections(user, API_FOLLOWERS, 1).then(function()
+                for(var i = 0; i < data.length; i++)
                 {
-                    updateProgress();
-                    resolve();
-                })
-            })
-        }
-        else
-        {
-            processConnections(user, API_FOLLOWING, 1).then(function()
+                    addConnection(user, data[i])
+                }
+                resolve();
+            }, (error)=>
             {
-                updateProgress();
+                console.log(error);
                 resolve();
             })
-        }
     });
 }
 
@@ -228,7 +181,7 @@ function processUserConnections(user)
  */
 function createConnections()
 {
-    return new Promise(function(resolve, reject)
+    return new Promise((resolve, reject)=>
     {
         var prom = [];
         for(var i = 0; i < nodes.length; i++)
@@ -248,6 +201,9 @@ function createConnections()
 }
 
 
+/**
+ * Updates progress bar for loading the JS graph
+ */
 function updateProgress()
 {
     indexed++;
@@ -267,16 +223,16 @@ function updateProgress()
  */
 function addSelfToGraph(username)
 {
-    return new Promise(function(resolve, reject)
+    return new Promise((resolve, reject)=>
     {
-        queryAPIByUser("", username, function(data)
+        queryAPIByUser("", username, (data)=>
         {
             baseID = data.id;
             total = (data.followers + data.following);
             addPersonToGraph(data);
             resolve();
         },
-        function(error)
+        (error)=>
         {
            reject(error);
         });
@@ -313,35 +269,32 @@ function createFriendsGraph(username, containerName, progressBarID)
 
     nodes = [];
     edges = [];
-    addSelfToGraph(username).then(function()
+    addSelfToGraph(username).then(()=>
     {
-        addFriends(username, API_FOLLOWERS,1).then(function()
+        addFriends(username).then(()=>
         {
-            addFriends(username, API_FOLLOWING,1).then(function()
+            createConnections().then(()=>
             {
-                createConnections().then(function()
-                {
-                    $("#" + progressID).html("");
+                $("#" + progressID).html("");
 
-                    var container = document.getElementById(containerName);
-                    var data =
-                        {
-                            nodes: nodes,
-                            edges: edges
-                        };
-                    var network = new vis.Network(container, data, options);
+                var container = document.getElementById(containerName);
+                var data =
+                    {
+                        nodes: nodes,
+                        edges: edges
+                    };
+                var network = new vis.Network(container, data, options);
 
-                    network.on("click", function (params)
+                network.on("click", function (params)
+                {
+                    if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN)
                     {
-                        if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN)
-                        {
-                            bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM)));
-                        }
-                    });
+                        bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM)));
+                    }
                 });
             });
         })
-    }).catch(function(error)
+    }).catch((error)=>
     {
         $("#" + graphsTitle).html("Error Fetching Data From API");
         alert("Invalid User");

public/js/githubAPI.js

@@ -17,10 +17,6 @@ const API_ORG_MEMBERS = "/members";
 
 const API_REPOS = "/repos";
 
-const API_FOLLOWING = "/following";
-
-const API_FOLLOWERS = "/followers";
-
 const API_REPOSITORIES = "/repos";
 
 const API_ORGANIZATIONS = "/orgs";
@@ -31,39 +27,72 @@ const API_ORGANIZATIONS = "/orgs";
  * allows you to get at the data using a
  * callback which gives you a json object.
  *
- * @param apiPath the path on the github api ie API_FOLLOWING
  * @param user the username to query
  * @param successCallBack callback to complete when data is returned
  * @param errorCallBack callback which is invoked on error
  */
-function queryAPIByUser(apiPath, user, successCallBack, errorCallBack) {
+function queryAPIByUser(apiPath, user, successCallBack, errorCallBack) 
+{
     const urlpath = APIROOT + API_USER_PATH + user + apiPath;
-    console.log(urlpath);
-    $.ajax({
-        type:'GET',
-        url: urlpath,
-        dataType: "json",
-        success: successCallBack,
-        error:errorCallBack,
-        timeout: 4000
-    });
+    runAjax(urlpath, successCallBack, errorCallBack);
 }
 
-function queryAPIByOrg(apiPath, org, successCallBack, errorCallBack) {
+/**
+ * Makes API calls for orgs on github
+ * 
+ * @param {*} apiPath 
+ * @param {*} org 
+ * @param {*} successCallBack 
+ * @param {*} errorCallBack 
+ */
+function queryAPIByOrg(apiPath, org, successCallBack, errorCallBack)
+{
     const urlpath = APIROOT + API_ORG_PATH + org + apiPath;
-    console.log(urlpath);
-    $.ajax({
-        type:'GET',
-        url: urlpath,
-        dataType: "json",
-        success: successCallBack,
-        error:errorCallBack,
-        timeout: 4000
-    });
+    runAjax(urlpath, successCallBack, errorCallBack);
 }
 
-function queryUrl(url, successCallBack, errorCallBack) {
+
+/**
+ * Fetches a list of fiends for a user.
+ * 
+ * @param {*} userName 
+ * @param {*} suc 
+ * @param {*} err 
+ */
+function getFriendsAPI(userName, suc, err)
+{
+    // api/friends/jrtechs
+    const urlpath = APIROOT + "/friends/" + userName;
+    runAjax(urlpath, suc, err);
+}
+
+
+/**
+ * Queries github API end points with the backend
+ * proxy server for github graphs.
+ * 
+ * @param {*} url 
+ * @param {*} successCallBack 
+ * @param {*} errorCallBack 
+ */
+function queryUrl(url, successCallBack, errorCallBack)
+{
     url = url.split("https://api.github.com/").join("api/");
+    runAjax(url, successCallBack, errorCallBack);
+}
+
+
+/**
+ * Wrapper for AJAX calls so we can unify
+ * all of our settings.
+ * 
+ * @param {*} url -- url to query
+ * @param {*} successCallBack  -- callback with data retrieved
+ * @param {*} errorCallBack  -- callback with error message
+ */
+function runAjax(url, successCallBack, errorCallBack)
+{
+    console.log(url);
     $.ajax({
         type:'GET',
         url: url,

routes/api.js

@@ -6,50 +6,226 @@ const GITHUB_API = "https://api.github.com";
 const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`;
 
 
+/**
+ * Queries data from the github APi server and returns it as
+ * a json object in a promise.
+ * 
+ * This makes no attempt to cache
+ * 
+ * @param {*} requestURL endpoint on githubapi: ex: /users/jrtechs/following
+ */
+function queryGithubAPIRaw(requestURL)
+{
+    return new Promise((resolve, reject)=>
+    {
+        var queryURL;
+        if(requestURL.includes("?page="))
+        {
+            queryURL = GITHUB_API + requestURL + "&" + authenticate;
+        }
+        else
+        {
+            queryURL = GITHUB_API + requestURL + "?" + authenticate;
+        }
+        console.log(queryURL);
+
+        got(queryURL, { json: true }).then(response =>
+        {
+            resolve(response.body);
+            cache.put(requestURL, response.body);
+        }).catch(error =>
+        {
+            resolve(error);
+            cache.put(requestURL, error);
+        });
+    });
+}
+
+
+/**
+ * Queries data from the github api server
+ * and caches the results locally.
+ * 
+ * @param {*} requestURL 
+ */
 function queryGitHubAPI(requestURL)
 {
     const apiData = cache.get(requestURL);
 
-    return new Promise(function(reject, resolve)
+    return new Promise(function(resolve, reject)
     {
         if(apiData == null)
         {
-            var queryURL;
-            if(requestURL.includes("?page="))
+            queryGithubAPIRaw(requestURL).then((dd)=>
             {
-                queryURL = GITHUB_API + requestURL + "&" + authenticate;
+                resolve(dd);
+            }).catch((err)=>
+            {
+                resolve(err);
+            })
+        }
+        else
+        {
+            console.log("Fetched From Cache");
+            resolve(apiData);
+        }
+    })
+}
+
+
+const API_FOLLOWING = "/following";
+const API_FOLLOWERS = "/followers";
+const API_USER_PATH = "/users/";
+const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
+// if this is too large, it would be infeasible to make graphs for people following popular people
+const API_MAX_PAGES = 3;
+const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
+
+
+/**
+ * This will fetch all of the either followers or following of a
+ * github user.
+ * 
+ * This function is recursive to traverse all the pagination so we
+ * can get a complete list of all the friends. The max amount of 
+ * followers/following you can get at once is 100.
+ * 
+ * @param {*} username username of github client
+ * @param {*} apiPath following or followers
+ * @param {*} page current pagination page
+ * @param {*} lst list we are building on
+ */
+function fetchAllUsers(username, apiPath, page, lst)
+{
+    return new Promise((resolve, reject)=>
+    {
+        queryGithubAPIRaw(API_USER_PATH + username + apiPath + "?page=" + page + API_PAGINATION).then((data)=>
+        {
+            if(data.hasOwnProperty("length"))
+            {
+                lst = lst.concat(data)
+                if(page < API_MAX_PAGES && data.length === API_PAGINATION_SIZE)
+                {
+                    fetchAllUsers(username, apiPath, page + 1, lst).then((l)=>
+                    {
+                        resolve(l);
+                    });
+                }
+                else
+                {
+                    resolve(lst);
+                }
             }
             else
             {
-                queryURL = GITHUB_API + requestURL + "?" + authenticate;
+                reject("Malformed data");
             }
-            console.log(queryURL);
+        }).catch((err)=>
+        {
+            reject("error with api request");
+        });
+    },
+    (error)=>
+    {
+        if(error.hasOwnProperty("length"))
+        {
+            lst.concat(data);
+            resolve(lst);
+        }
+    });
+}
+
+
+/**
+ * Combines the list of friends and followers ignoring duplicates
+ * that are already in the list. (person is both following and followed by someone)
+ * 
+ * This also removes any unused properties like events_url and organizations_url
+ * 
+ * @param {*} followingAndFollowers 
+ */
+function minimizeFriends(people)
+{
+    var friendLst = [];
+
+    var ids = new Set();
+
+    for(var i = 0; i < people.length; i++)
+    {
+        if(!ids.has(people[i].id))
+        {
+            ids.add(people[i].id);
+            friendLst.push({
+                id: people[i].id,
+                 login: people[i].login, 
+                 avatar_url: people[i].avatar_url
+            });
+        }
+    }
+    return friendLst;
+}
 
-            got(queryURL, { json: true }).then(response =>
+
+/**
+ * Fetches all the people that are either following or is followed
+ * by a person on github. This will cache the results to make simultaneous
+ * connections easier and less demanding on the github API.
+ * 
+ * @param {*} user 
+ */
+function queryFriends(user)
+{
+    const cacheHit = cache.get("/friends/" + user);
+    return new Promise((resolve, reject)=>
+    {
+        if(cacheHit == null)
+        {
+            fetchAllUsers(user, API_FOLLOWERS, 1, []).then((followers)=>
             {
-                resolve(response.body);
-                cache.put(requestURL, response.body);
-            }).catch(error =>
+                fetchAllUsers(user, API_FOLLOWING, 1, []).then((following)=>
+                {
+                    var fList = minimizeFriends(following.concat(followers));
+                    resolve(fList);
+                    cache.put("/friends/" + user, fList);
+                }).catch((err)=>
+                {
+                    console.log(err);  
+                })
+            }).catch((error)=>
             {
-                resolve(error);
-                cache.put(requestURL, error);
-            });
+                console.log(error);
+            })
         }
         else
         {
-            console.log("Fetched From Cahce");
-            resolve(apiData);
+            console.log("Friends cache hit");
+            resolve(cacheHit);
         }
-    })
+    });
 }
 
 
+routes.get("/friends/:name", (request, result)=>
+{
+    queryFriends(request.params.name).then(friends=>
+    {
+        result.json(friends)
+            .end();
+    }).catch(error=>
+    {
+        result.status(500)
+            .json({error: 'API error fetching friends'})
+            .end();
+    });
+})
+
+
 routes.get('/*', (request, result) =>
 {
     var gitHubAPIURL = request.url;
 
     result.setHeader('Content-Type', 'application/json');
-    queryGitHubAPI(gitHubAPIURL).then(function(data)
+    queryGitHubAPI(gitHubAPIURL).then((data)=>
     {
         if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id"))
         {
@@ -60,7 +236,7 @@ routes.get('/*', (request, result) =>
             result.write("[]");
         }
         result.end();
-    }).catch(function(error)
+    }).catch((error)=>
     {
         try
         {
@@ -74,7 +250,8 @@ routes.get('/*', (request, result) =>
             }
 
         }
-        catch(error) {
+        catch(error) 
+        {
             result.write("[]");
         };
         result.end();