Browse Source

Updated org repositories to use new minimized cached system on API.

performance-improvements
jrtechs 4 years ago
parent
commit
7c4890b23a
3 changed files with 107 additions and 70 deletions
  1. +12
    -19
      public/js/createOrgTable.js
  2. +10
    -0
      public/js/githubAPI.js
  3. +85
    -51
      routes/api.js

+ 12
- 19
public/js/createOrgTable.js View File

@ -16,29 +16,22 @@ function generateHtmlRow(repoData) {
var repos = []; var repos = [];
function fetchAllRepositories(orgName, page)
function fetchAllRepositories(orgName)
{ {
return new Promise(function(resolve, reject)
console.log("Going for it");
return new Promise((resolve, reject)=>
{ {
queryAPIByOrg(API_REPOSITORIES + "?page=" + page, orgName,
function(data)
getOrganizationRepositories(orgName,
(data)=>
{ {
console.log("Dam did got it");
repos.push(...data); repos.push(...data);
if (data.length === 30)
{
fetchAllRepositories(orgName, page + 1).then(function ()
{
resolve();
})
}
else {
resolve();
}
resolve();
}, },
function(error)
(error)=>
{ {
//console.log("Unable to load table data");
console.log("Unable to load table data");
reject("Error fetching repositories");
}); });
}); });
} }
@ -48,7 +41,7 @@ function createOrgTable(orgName, tableContainer)
{ {
var html = ""; var html = "";
fetchAllRepositories(orgName, 1).then(function() {
fetchAllRepositories(orgName).then(function() {
for (var i=0; i < repos.length; i++) { for (var i=0; i < repos.length; i++) {
let icon = repos[i].language; let icon = repos[i].language;
icon === null icon === null
@ -81,6 +74,6 @@ function createOrgTable(orgName, tableContainer)
}, 1500); }, 1500);
}).catch(function(error) }).catch(function(error)
{ {
//console.log("Unable to create table");
console.log("Unable to create table");
}); });
} }

+ 10
- 0
public/js/githubAPI.js View File

@ -75,6 +75,16 @@ function getUserRepositories(userName, suc, err)
} }
function getOrganizationRepositories(orgName, suc, err)
{
//ex: http://localhost:7000/api/org/repositories/ComputerScienceHouse
const urlpath = APIROOT + "/org/repositories/" + orgName;
console.log("what is even happening rn.");
console.log(urlpath);
runAjax(urlpath, suc, err);
}
/** /**
* Queries github API end points with the backend * Queries github API end points with the backend
* proxy server for github graphs. * proxy server for github graphs.

+ 85
- 51
routes/api.js View File

@ -5,6 +5,17 @@ const dotenv = require("dotenv").config();
const GITHUB_API = "https://api.github.com"; const GITHUB_API = "https://api.github.com";
const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`; const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`;
const API_FOLLOWING = "/following";
const API_FOLLOWERS = "/followers";
const API_USER_PATH = "/users/";
const API_ORGS_PATH = "/orgs/";
const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
// if this is too large, it would be infeasible to make graphs for people following popular people
const API_MAX_PAGES = 2;
const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
const REPOS_PATH = "/repos";
/** /**
* Queries data from the github APi server and returns it as * Queries data from the github APi server and returns it as
@ -73,40 +84,27 @@ function queryGitHubAPI(requestURL)
} }
const API_FOLLOWING = "/following";
const API_FOLLOWERS = "/followers";
const API_USER_PATH = "/users/";
const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
// if this is too large, it would be infeasible to make graphs for people following popular people
const API_MAX_PAGES = 2;
const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
/** /**
* This will fetch all of the either followers or following of a
* github user.
*
* This function is recursive to traverse all the pagination so we
* can get a complete list of all the friends. The max amount of
* followers/following you can get at once is 100.
* Fetches all content from a particular github api endpoint
* using their pagination schema.
* *
* @param {*} username username of github client * @param {*} username username of github client
* @param {*} apiPath following or followers * @param {*} apiPath following or followers
* @param {*} page current pagination page * @param {*} page current pagination page
* @param {*} lst list we are building on * @param {*} lst list we are building on
*/ */
function fetchAllUsers(username, apiPath, page, lst)
function fetchAllWithPagination(apiPath, page, lst)
{ {
return new Promise((resolve, reject)=> return new Promise((resolve, reject)=>
{ {
queryGithubAPIRaw(API_USER_PATH + username + apiPath + "?page=" + page + API_PAGINATION).then((data)=>
queryGithubAPIRaw(apiPath + "?page=" + page + API_PAGINATION).then((data)=>
{ {
if(data.hasOwnProperty("length")) if(data.hasOwnProperty("length"))
{ {
lst = lst.concat(data) lst = lst.concat(data)
if(page < API_MAX_PAGES && data.length === API_PAGINATION_SIZE) if(page < API_MAX_PAGES && data.length === API_PAGINATION_SIZE)
{ {
fetchAllUsers(username, apiPath, page + 1, lst).then((l)=>
fetchAllWithPagination(apiPath, page + 1, lst).then((l)=>
{ {
resolve(l); resolve(l);
}); });
@ -137,6 +135,23 @@ function fetchAllUsers(username, apiPath, page, lst)
} }
/**
* Makes a copy of a JS object with certain properties
*
* @param {*} props
* @param {*} obj
*/
function copyWithProperties(props, obj)
{
var newO = new Object();
for(var i =0; i < props.length; i++)
{
newO[props[i]] = obj[props[i]];
}
return newO;
}
/** /**
* Combines the list of friends and followers ignoring duplicates * Combines the list of friends and followers ignoring duplicates
* that are already in the list. (person is both following and followed by someone) * that are already in the list. (person is both following and followed by someone)
@ -180,9 +195,9 @@ function queryFriends(user)
{ {
if(cacheHit == null) if(cacheHit == null)
{ {
fetchAllUsers(user, API_FOLLOWERS, 1, []).then((followers)=>
fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWERS, 1, []).then((followers)=>
{ {
fetchAllUsers(user, API_FOLLOWING, 1, []).then((following)=>
fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWING, 1, []).then((following)=>
{ {
var fList = minimizeFriends(following.concat(followers)); var fList = minimizeFriends(following.concat(followers));
resolve(fList); resolve(fList);
@ -207,31 +222,11 @@ function queryFriends(user)
} }
routes.get("/friends/:name", (request, result)=>
{
queryFriends(request.params.name).then(friends=>
{
result.json(friends)
.end();
}).catch(error=>
{
result.status(500)
.json({error: 'API error fetching friends'})
.end();
});
});
function copyWithProperties(props, obj)
{
var newO = new Object();
for(var i =0; i < props.length; i++)
{
newO[props[i]] = obj[props[i]];
}
return newO;
}
/**
* Minimizes the JSON for a list of repositories
*
* @param {*} repositories
*/
function minimizeRepositories(repositories) function minimizeRepositories(repositories)
{ {
var rList = []; var rList = [];
@ -240,27 +235,36 @@ function minimizeRepositories(repositories)
{ {
rList.push(copyWithProperties(["name", "created_at", "homepage", rList.push(copyWithProperties(["name", "created_at", "homepage",
"description", "language", "forks", "watchers", "description", "language", "forks", "watchers",
"open_issues_count", "license"],
"open_issues_count", "license", "html_url"],
repositories[i])); repositories[i]));
} }
return rList; return rList;
} }
const REPOS_PATH = "/repos";
function queryRepositories(user)
/**
* Fetches all repositories from the API
*
* @param {*} user name of org/user
* @param {*} orgsOrUsers either /users/ or /orgs/
*/
function queryRepositories(user, orgsOrUsers)
{ {
const cacheHit = cache.get(user + REPOS_PATH); const cacheHit = cache.get(user + REPOS_PATH);
return new Promise((resolve, reject)=> return new Promise((resolve, reject)=>
{ {
if(cacheHit == null) if(cacheHit == null)
{ {
fetchAllUsers(user, REPOS_PATH, 1, []).then((repos)=>
fetchAllWithPagination(orgsOrUsers + user + REPOS_PATH, 1, []).then((repos)=>
{ {
var minimized = minimizeRepositories(repos); var minimized = minimizeRepositories(repos);
resolve(minimized); resolve(minimized);
cache.put(user + REPOS_PATH, minimized); cache.put(user + REPOS_PATH, minimized);
});
}).catch((err)=>
{
console.log(err)
console.log("bad things went down");
})
} }
else else
{ {
@ -271,9 +275,39 @@ function queryRepositories(user)
} }
routes.get("/friends/:name", (request, result)=>
{
queryFriends(request.params.name).then(friends=>
{
result.json(friends)
.end();
}).catch(error=>
{
result.status(500)
.json({error: 'API error fetching friends'})
.end();
});
});
routes.get("/repositories/:name", (request, result)=> routes.get("/repositories/:name", (request, result)=>
{ {
queryRepositories(request.params.name).then(repos=>
queryRepositories(request.params.name, API_USER_PATH).then(repos=>
{
result.json(repos)
.end();
}).catch(error=>
{
result.status(500)
.json({error: 'API error fetching friends'})
.end();
});
});
routes.get("/org/repositories/:name", (request, result)=>
{
queryRepositories(request.params.name, API_ORGS_PATH).then(repos=>
{ {
result.json(repos) result.json(repos)
.end(); .end();

Loading…
Cancel
Save