const routes = require('express').Router(); const got = require("got"); const cache = require('memory-cache'); const dotenv = require("dotenv").config(); const GITHUB_API = "https://api.github.com"; const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`; const API_FOLLOWING = "/following"; const API_FOLLOWERS = "/followers"; const API_USER_PATH = "/users/"; const API_ORGS_PATH = "/orgs/"; const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default // if this is too large, it would be infeasible to make graphs for people following popular people const API_MAX_PAGES = 2; const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE; const REPOS_PATH = "/repos"; const got_options = { json: true, username : process.env.CLIENT_ID, password : process.env.CLIENT_SECRET }; /** * Queries data from the github APi server and returns it as * a json object in a promise. * * This makes no attempt to cache * * @param {*} requestURL endpoint on githubapi: ex: /users/jrtechs/following */ function queryGithubAPIRaw(requestURL) { return new Promise((resolve, reject)=> { var queryURL; if(requestURL.includes("?page=")) { queryURL = GITHUB_API + requestURL + "&" + authenticate; } else { queryURL = GITHUB_API + requestURL + "?" + authenticate; } console.log(queryURL); got(queryURL, got_options).then(response => { resolve(response.body); cache.put(requestURL, response.body); }).catch(error => { resolve(error); cache.put(requestURL, error); }); }); } /** * Queries data from the github api server * and caches the results locally. * * @param {*} requestURL */ function queryGitHubAPI(requestURL) { const apiData = cache.get(requestURL); return new Promise(function(resolve, reject) { if(apiData == null) { queryGithubAPIRaw(requestURL).then((dd)=> { resolve(dd); }).catch((err)=> { resolve(err); }) } else { console.log("Fetched From Cache"); resolve(apiData); } }) } /** * Fetches all content from a particular github api endpoint * using their pagination schema. * * @param {*} username username of github client * @param {*} apiPath following or followers * @param {*} page current pagination page * @param {*} lst list we are building on */ function fetchAllWithPagination(apiPath, page, lst) { return new Promise((resolve, reject)=> { queryGithubAPIRaw(apiPath + "?page=" + page + API_PAGINATION).then((data)=> { if(data.hasOwnProperty("length")) { lst = lst.concat(data) if(page < API_MAX_PAGES && data.length === API_PAGINATION_SIZE) { fetchAllWithPagination(apiPath, page + 1, lst).then((l)=> { resolve(l); }); } else { resolve(lst); } } else { console.log(data); reject("Malformed data"); } }).catch((err)=> { reject("error with api request"); }); }, (error)=> { if(error.hasOwnProperty("length")) { lst.concat(data); resolve(lst); } }); } /** * Makes a copy of a JS object with certain properties * * @param {*} props * @param {*} obj */ function copyWithProperties(props, obj) { var newO = new Object(); for(var i =0; i < props.length; i++) { newO[props[i]] = obj[props[i]]; } return newO; } /** * Combines the list of friends and followers ignoring duplicates * that are already in the list. (person is both following and followed by someone) * * This also removes any unused properties like events_url and organizations_url * * @param {*} followingAndFollowers */ function minimizeFriends(people) { var friendLst = []; var ids = new Set(); for(var i = 0; i < people.length; i++) { if(!ids.has(people[i].id)) { ids.add(people[i].id); friendLst.push({ login: people[i].login, avatar_url: people[i].avatar_url }); } } return friendLst; } /** * Fetches all the people that are either following or is followed * by a person on github. This will cache the results to make simultaneous * connections easier and less demanding on the github API. * * @param {*} user */ function queryFriends(user) { const cacheHit = cache.get("/friends/" + user); return new Promise((resolve, reject)=> { if(cacheHit == null) { fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWERS, 1, []).then((followers)=> { fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWING, 1, []).then((following)=> { var fList = minimizeFriends(following.concat(followers)); resolve(fList); cache.put("/friends/" + user, fList); }).catch((err)=> { console.log(err); reject("API ERROR"); }) }).catch((error)=> { console.log(error); resolve("API Error"); }) } else { console.log("Friends cache hit"); resolve(cacheHit); } }); } /** * * Fetches all of the members of an organization from the * API or cache * * /orgs/RITlug/members?page=1 * * @param {*} orgName */ function getOrganizationMembers(orgName) { const cacheHit = cache.get("/org/users/" + orgName); return new Promise((resolve, reject)=> { if(cacheHit == null) { fetchAllWithPagination(API_ORGS_PATH + orgName + "/members", 1, []).then((mems)=> { var minimized = minimizeFriends(mems); resolve(minimized); cache.put("/org/users/" + orgName, minimized); }).catch((err)=> { console.log(err) }) } else { console.log("Org members cache hit"); resolve(cacheHit); } }); } /** * Minimizes the JSON for a list of repositories * * @param {*} repositories */ function minimizeRepositories(repositories) { var rList = []; for(var i = 0; i < repositories.length; i++) { rList.push(copyWithProperties(["name", "created_at", "homepage", "description", "language", "forks", "watchers", "open_issues_count", "license", "html_url"], repositories[i])); } return rList; } /** * Fetches all repositories from the API * * @param {*} user name of org/user * @param {*} orgsOrUsers either /users/ or /orgs/ */ function queryRepositories(user, orgsOrUsers) { const cacheHit = cache.get(user + REPOS_PATH); return new Promise((resolve, reject)=> { if(cacheHit == null) { fetchAllWithPagination(orgsOrUsers + user + REPOS_PATH, 1, []).then((repos)=> { var minimized = minimizeRepositories(repos); resolve(minimized); cache.put(user + REPOS_PATH, minimized); }).catch((err)=> { console.log(err) console.log("bad things went down"); }) } else { console.log("Repositories cache hit"); resolve(cacheHit); } }); } /** * /users/name/following/followers */ routes.get("/friends/:name", (request, result)=> { queryFriends(request.params.name).then(friends=> { result.json(friends) .end(); }).catch(error=> { result.status(500) .json({error: 'API error fetching friends'}) .end(); }); }); routes.get("/org/users/:name", (request, result)=> { getOrganizationMembers(request.params.name).then(friends=> { result.json(friends) .end(); }).catch(error=> { result.status(500) .json({error: 'API error fetching friends'}) .end(); }); }); routes.get("/repositories/:name", (request, result)=> { queryRepositories(request.params.name, API_USER_PATH).then(repos=> { result.json(repos) .end(); }).catch(error=> { result.status(500) .json({error: 'API error fetching friends'}) .end(); }); }); routes.get("/org/repositories/:name", (request, result)=> { queryRepositories(request.params.name, API_ORGS_PATH).then(repos=> { result.json(repos) .end(); }).catch(error=> { result.status(500) .json({error: 'API error fetching friends'}) .end(); }); }); routes.get('/*', (request, result) => { var gitHubAPIURL = request.url; result.setHeader('Content-Type', 'application/json'); queryGitHubAPI(gitHubAPIURL).then((data)=> { if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id")) { result.write(JSON.stringify(data)); } else { result.write("[]"); } result.end(); }).catch((error)=> { try { if(error.hasOwnProperty("id") || error[0].hasOwnProperty("id")) { result.write(JSON.stringify(error)); } else { result.write("[]"); } } catch(error) { result.write("[]"); }; result.end(); }); if(cache.size() > 50000) { cache.clear(); } }); module.exports = routes;