Merge pull request #27 from jrtechs/performance-improvements

Performance Improvements

public/js/createOrgRepoGraph.js

@@ -6,31 +6,19 @@
  * @param page
  * @returns {Promise<any>}
  */
-function addOrgUsers(orgname, page)
+function addOrgUsers(orgname)
 {
     return new Promise(function(resolve, reject)
     {
-        queryAPIByOrg(API_ORG_MEMBERS + "?page=" + page, orgname, function(data)
+        getOrganizationMembers(orgname, (data)=>
         {
             for(var i = 0;i < data.length; i++)
             {
                 addPersonToGraph(data[i]);
             }
-
-            if(data.length === 30)
-            {
-                addOrgUsers(orgname, page + 1).then(function()
-                {
-                    resolve();
-                });
-            }
-            else
-            {
-                total = 2*(data.length + (page * 30));
-                resolve();
-            }
-
-        }, function(error)
+            total = data.length;
+            resolve();
+        }, (error)=>
         {
             console.log(error);
             resolve();
@@ -38,6 +26,7 @@ function addOrgUsers(orgname, page)
     })
 }
 
+
 /**
  * Creates a graph
  * @param username
@@ -51,7 +40,7 @@ function createOrgRepoGraph(orgname, containerName, graphsTitle)
     nodes = [];
     edges = [];
 
-    addOrgUsers(orgname, 1).then(function()
+    addOrgUsers(orgname).then(function()
     {
         createConnections().then( () => {
             var container = document.getElementById(containerName);
@@ -62,8 +51,9 @@ function createOrgRepoGraph(orgname, containerName, graphsTitle)
             var network = new vis.Network(container, data, options);
             network.on("click", function (params) {
                 params.event = "[original event]";
-                if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN) {
-                    bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM)));
+                if(this.getNodeAt(params.pointer.DOM) !== NaN) 
+                {
+                    bringUpProfileView(this.getNodeAt(params.pointer.DOM));
                 }
             });
 

public/js/createOrgTable.js

@@ -16,29 +16,22 @@ function generateHtmlRow(repoData) {
 
 var repos = [];
 
-function fetchAllRepositories(orgName, page)
+function fetchAllRepositories(orgName)
 {
-    return new Promise(function(resolve, reject)
+    console.log("Going for it");
+    return new Promise((resolve, reject)=>
     {
-        queryAPIByOrg(API_REPOSITORIES + "?page=" + page, orgName,
-            function(data)
+        getOrganizationRepositories(orgName,
+            (data)=>
             {
+                console.log("Dam did got it");
                 repos.push(...data);
-
-                if (data.length === 30)
-                {
-                    fetchAllRepositories(orgName, page + 1).then(function ()
-                    {
-                        resolve();
-                    })
-                }
-                else {
-                    resolve();
-                }
+                resolve();
             },
-            function(error)
+            (error)=>
             {
-                //console.log("Unable to load table data");
+                console.log("Unable to load table data");
+                reject("Error fetching repositories");
             });
     });
 }
@@ -48,7 +41,7 @@ function createOrgTable(orgName, tableContainer)
 {
     var html = "";
 
-    fetchAllRepositories(orgName, 1).then(function() {
+    fetchAllRepositories(orgName).then(function() {
         for (var i=0; i < repos.length; i++) {
             let icon = repos[i].language;
             icon === null
@@ -81,6 +74,6 @@ function createOrgTable(orgName, tableContainer)
         }, 1500);
     }).catch(function(error)
     {
-        //console.log("Unable to create table");
+        console.log("Unable to create table");
     });
 }

public/js/friendsGraph.js

@@ -37,11 +37,11 @@ var options = {
  * @param userID
  * @returns {boolean}
  */
-function alreadyInGraph(userID)
+function alreadyInGraph(username)
 {
     for(var i = 0; i < nodes.length; i++)
     {
-        if(nodes[i].id === userID)
+        if(nodes[i].id === username)
         {
             return true;
         }
@@ -59,8 +59,7 @@ function addPersonToGraph(profileData)
 {
     nodes.push(
         {
-            id:profileData.id,
-            name:profileData.login,
+            id:profileData.login,
             shape: 'circularImage',
             image:profileData.avatar_url
         });
@@ -84,7 +83,7 @@ function addFriends(username)
         {
             for(var i = 0; i < data.length; i++)
             {
-                if(!alreadyInGraph(data[i].id))
+                if(!alreadyInGraph(data[i].login))
                 {
                     addPersonToGraph(data[i]);
                 }
@@ -110,11 +109,8 @@ function edgeInGraph(id1, id2)
 {
     for(var i = 0;i < edges.length; i++)
     {
-        if(edges[i].from === id1 && edges[i].to === id2)
-        {
-            return true;
-        }
-        if(edges[i].to === id1 && edges[i].from === id2)
+        if((edges[i].to === id1 && edges[i].from === id2) || 
+            (edges[i].from === id1 && edges[i].to === id2))
         {
             return true;
         }
@@ -131,14 +127,14 @@ function edgeInGraph(id1, id2)
  */
 function addConnection(person1, person2)
 {
-    if(person1.id !== person2.id)
+    if(person1.login !== person2.login)
     {
-        if(alreadyInGraph(person2.id) && !edgeInGraph(person1.id, person2.id))
+        if(alreadyInGraph(person2.login) && !edgeInGraph(person1.id, person2.login))
         {
             edges.push(
                 {
                     from: person1.id,
-                    to: person2.id
+                    to: person2.login
                 });
         }
     }
@@ -156,7 +152,7 @@ function processUserConnections(user)
     return new Promise(function(resolve, reject)
     {
         updateProgress();
-        getFriendsAPI(user.name,
+        getFriendsAPI(user.id,
             (data)=>
             {
                 for(var i = 0; i < data.length; i++)
@@ -168,7 +164,7 @@ function processUserConnections(user)
             {
                 console.log(error);
                 resolve();
-            })
+            });
     });
 }
 
@@ -227,7 +223,7 @@ function addSelfToGraph(username)
     {
         queryAPIByUser("", username, (data)=>
         {
-            baseID = data.id;
+            baseID = data.login;
             total = (data.followers + data.following);
             addPersonToGraph(data);
             resolve();
@@ -245,13 +241,13 @@ function addSelfToGraph(username)
  *
  * @param github id
  */
-function bringUpProfileView(id)
+function bringUpProfileView(uname)
 {
     for(var i = 0; i < nodes.length; i++)
     {
-        if(nodes[i].id === id)
+        if(nodes[i].id === uname)
         {
-            profileGen(nodes[i].name, "profileGen");
+            profileGen(nodes[i].id, "profileGen");
         }
     }
 }
@@ -289,7 +285,7 @@ function createFriendsGraph(username, containerName, progressBarID)
                 {
                     if(Number(this.getNodeAt(params.pointer.DOM)) !== NaN)
                     {
-                        bringUpProfileView(Number(this.getNodeAt(params.pointer.DOM)));
+                        bringUpProfileView(this.getNodeAt(params.pointer.DOM));
                     }
                 });
             });

public/js/githubAPI.js

@@ -67,6 +67,29 @@ function getFriendsAPI(userName, suc, err)
 }
 
 
+function getUserRepositories(userName, suc, err)
+{
+    //ex: http://localhost:7000/api/repositories/jwflory
+    const urlpath = APIROOT + "/repositories/" + userName;
+    runAjax(urlpath, suc, err);
+}
+
+
+function getOrganizationRepositories(orgName, suc, err)
+{
+    //ex: http://localhost:7000/api/org/repositories/ComputerScienceHouse
+    const urlpath = APIROOT + "/org/repositories/" + orgName;
+    runAjax(urlpath, suc, err);
+}
+
+
+function getOrganizationMembers(orgName, suc, err)
+{
+    const urlpath = APIROOT + "/org/users/" + orgName;
+    runAjax(urlpath, suc, err);
+}
+
+
 /**
  * Queries github API end points with the backend
  * proxy server for github graphs.

public/js/profileTimeLine.js

@@ -15,7 +15,7 @@ function addRepositories(userName, groupID)
 {
     return new Promise(function(resolve, reject)
     {
-        queryAPIByUser(API_REPOSITORIES, userName,
+        getUserRepositories(userName,
             function(data)
             {
                 repositoryData = data;
@@ -76,7 +76,6 @@ function createProfileTimeLine(username, containerName)
 {
     var container = document.getElementById(containerName);
 
-
     var prom = [addRepositories(username, 1)];
 
     var groups = new vis.DataSet([

routes/api.js

@@ -5,6 +5,17 @@ const dotenv = require("dotenv").config();
 const GITHUB_API = "https://api.github.com";
 const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`;
 
+const API_FOLLOWING = "/following";
+const API_FOLLOWERS = "/followers";
+const API_USER_PATH = "/users/";
+const API_ORGS_PATH = "/orgs/";
+const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
+// if this is too large, it would be infeasible to make graphs for people following popular people
+const API_MAX_PAGES = 2;
+const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
+
+const REPOS_PATH = "/repos";
+
 
 /**
  * Queries data from the github APi server and returns it as
@@ -73,40 +84,27 @@ function queryGitHubAPI(requestURL)
 }
 
 
-const API_FOLLOWING = "/following";
-const API_FOLLOWERS = "/followers";
-const API_USER_PATH = "/users/";
-const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
-// if this is too large, it would be infeasible to make graphs for people following popular people
-const API_MAX_PAGES = 3;
-const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
-
-
 /**
- * This will fetch all of the either followers or following of a
- * github user.
- * 
- * This function is recursive to traverse all the pagination so we
- * can get a complete list of all the friends. The max amount of 
- * followers/following you can get at once is 100.
+ * Fetches all content from a particular github api endpoint
+ * using their pagination schema.
  * 
  * @param {*} username username of github client
  * @param {*} apiPath following or followers
  * @param {*} page current pagination page
  * @param {*} lst list we are building on
  */
-function fetchAllUsers(username, apiPath, page, lst)
+function fetchAllWithPagination(apiPath, page, lst)
 {
     return new Promise((resolve, reject)=>
     {
-        queryGithubAPIRaw(API_USER_PATH + username + apiPath + "?page=" + page + API_PAGINATION).then((data)=>
+        queryGithubAPIRaw(apiPath + "?page=" + page + API_PAGINATION).then((data)=>
         {
             if(data.hasOwnProperty("length"))
             {
                 lst = lst.concat(data)
                 if(page < API_MAX_PAGES && data.length === API_PAGINATION_SIZE)
                 {
-                    fetchAllUsers(username, apiPath, page + 1, lst).then((l)=>
+                    fetchAllWithPagination(apiPath, page + 1, lst).then((l)=>
                     {
                         resolve(l);
                     });
@@ -118,6 +116,7 @@ function fetchAllUsers(username, apiPath, page, lst)
             }
             else
             {
+                console.log(data);
                 reject("Malformed data");
             }
         }).catch((err)=>
@@ -136,6 +135,23 @@ function fetchAllUsers(username, apiPath, page, lst)
 }
 
 
+/**
+ * Makes a copy of a JS object with certain properties
+ * 
+ * @param {*} props 
+ * @param {*} obj 
+ */
+function copyWithProperties(props, obj)
+{
+    var newO = new Object();
+    for(var i =0; i < props.length; i++)
+    {
+        newO[props[i]] = obj[props[i]];
+    }
+    return newO;
+}
+
+
 /**
  * Combines the list of friends and followers ignoring duplicates
  * that are already in the list. (person is both following and followed by someone)
@@ -156,9 +172,8 @@ function minimizeFriends(people)
         {
             ids.add(people[i].id);
             friendLst.push({
-                id: people[i].id,
-                 login: people[i].login, 
-                 avatar_url: people[i].avatar_url
+                login: people[i].login, 
+                avatar_url: people[i].avatar_url
             });
         }
     }
@@ -180,9 +195,9 @@ function queryFriends(user)
     {
         if(cacheHit == null)
         {
-            fetchAllUsers(user, API_FOLLOWERS, 1, []).then((followers)=>
+            fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWERS, 1, []).then((followers)=>
             {
-                fetchAllUsers(user, API_FOLLOWING, 1, []).then((following)=>
+                fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWING, 1, []).then((following)=>
                 {
                     var fList = minimizeFriends(following.concat(followers));
                     resolve(fList);
@@ -190,10 +205,12 @@ function queryFriends(user)
                 }).catch((err)=>
                 {
                     console.log(err);  
+                    reject("API ERROR");
                 })
             }).catch((error)=>
             {
                 console.log(error);
+                resolve("API Error");
             })
         }
         else
@@ -205,6 +222,97 @@ function queryFriends(user)
 }
 
 
+/**
+ * 
+ * Fetches all of the members of an organization from the
+ * API or cache
+ *
+ * /orgs/RITlug/members?page=1
+ *
+ * @param {*} orgName 
+ */
+function getOrganizationMembers(orgName)
+{
+    const cacheHit = cache.get("/org/users/" + orgName);
+    return new Promise((resolve, reject)=>
+    {
+        if(cacheHit == null)
+        {
+            fetchAllWithPagination(API_ORGS_PATH + orgName + "/members", 1, []).then((mems)=>
+            {
+                var minimized = minimizeFriends(mems);
+                resolve(minimized);
+                cache.put("/org/users/" + orgName, minimized);
+            }).catch((err)=>
+            {
+                console.log(err)
+            })
+        }
+        else
+        {
+            console.log("Org members cache hit");
+            resolve(cacheHit);
+        }
+    });
+}
+
+
+/**
+ * Minimizes the JSON for a list of repositories
+ * 
+ * @param {*} repositories 
+ */
+function minimizeRepositories(repositories)
+{
+    var rList = [];
+
+    for(var i = 0; i < repositories.length; i++)
+    {
+        rList.push(copyWithProperties(["name", "created_at", "homepage", 
+            "description", "language", "forks", "watchers",
+             "open_issues_count", "license", "html_url"],
+            repositories[i]));
+    }
+    return rList;
+}
+
+
+/**
+ * Fetches all repositories from the API
+ * 
+ * @param {*} user name of org/user
+ * @param {*} orgsOrUsers  either /users/ or /orgs/
+ */
+function queryRepositories(user, orgsOrUsers)
+{
+    const cacheHit = cache.get(user + REPOS_PATH);
+    return new Promise((resolve, reject)=>
+    {
+        if(cacheHit == null)
+        {
+            fetchAllWithPagination(orgsOrUsers + user + REPOS_PATH, 1, []).then((repos)=>
+            {
+                var minimized = minimizeRepositories(repos);
+                resolve(minimized);
+                cache.put(user + REPOS_PATH, minimized);
+            }).catch((err)=>
+            {
+                console.log(err)
+                console.log("bad things went down");
+            })
+        }
+        else
+        {
+            console.log("Repositories cache hit");
+            resolve(cacheHit);
+        }
+    });
+}
+
+
+/**
+ * /users/name/following/followers
+ */
 routes.get("/friends/:name", (request, result)=>
 {
     queryFriends(request.params.name).then(friends=>
@@ -217,7 +325,54 @@ routes.get("/friends/:name", (request, result)=>
             .json({error: 'API error fetching friends'})
             .end();
     });
-})
+});
+
+
+
+routes.get("/org/users/:name", (request, result)=>
+{
+    getOrganizationMembers(request.params.name).then(friends=>
+        {
+            result.json(friends)
+                .end();
+        }).catch(error=>
+        {
+            result.status(500)
+                .json({error: 'API error fetching friends'})
+                .end();
+        });
+});
+
+
+
+routes.get("/repositories/:name", (request, result)=>
+{
+    queryRepositories(request.params.name, API_USER_PATH).then(repos=>
+        {
+            result.json(repos)
+                .end();
+        }).catch(error=>
+        {
+            result.status(500)
+                .json({error: 'API error fetching friends'})
+                .end();
+        });
+});
+
+
+routes.get("/org/repositories/:name", (request, result)=>
+{
+    queryRepositories(request.params.name, API_ORGS_PATH).then(repos=>
+        {
+            result.json(repos)
+                .end();
+        }).catch(error=>
+        {
+            result.status(500)
+                .json({error: 'API error fetching friends'})
+                .end();
+        });
+});
 
 
 routes.get('/*', (request, result) =>