API Rework (#28)

API refactor

routes/api/v2.js

@@ -0,0 +1,310 @@
+const routes = require('express').Router();
+const got = require("got");
+const cache = require('memory-cache');
+const dotenv = require("dotenv").config();
+const GITHUB_API = "https://api.github.com";
+const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`;
+
+const API_FOLLOWING = "/following";
+const API_FOLLOWERS = "/followers";
+const API_USER_PATH = "/users/";
+const API_ORGS_PATH = "/orgs/";
+const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
+// if this is too large, it would be infeasible to make graphs for people following popular people
+const API_MAX_PAGES = 2;
+const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
+
+const REPOS_PATH = "/repos";
+
+
+/**
+ * Queries data from the github APi server and returns it as
+ * a json object in a promise.
+ * 
+ * This makes no attempt to cache
+ * 
+ * @param {*} requestURL endpoint on githubapi: ex: /users/jrtechs/following
+ */
+const queryGithubAPIRaw = async requestURL => {
+    let queryURL = requestURL.includes("?page=") ? `${GITHUB_API}${requestURL}&${authenticate}` :`${GITHUB_API}${requestURL}?${authenticate}`;
+    console.log(queryURL);
+    try {
+        const req = await got(queryURL, { json: true });
+        cache.put(requestURL, req);
+        return req;
+    } catch (error) {
+        console.log(error);
+        cache.put(requestURL, `${error.statusCode} - ${error.statusMessage}`);
+    }
+}
+
+
+/**
+ * Queries data from the github api server
+ * and caches the results locally.
+ * 
+ * @param {*} requestURL 
+ */
+const queryGitHubAPI = async requestURL => {
+    const apiData = cache.get(requestURL);
+    if (apiData) {
+        console.log("Fetched From Cache");
+        return apiData
+    }
+
+    try {
+        return await queryGithubAPIRaw(requestURL);
+    } catch (error) {
+        console.log(error);
+    }
+}
+
+
+/**
+ * Fetches all content from a particular github api endpoint
+ * using their pagination schema.
+ * 
+ * @param {*} username username of github client
+ * @param {*} apiPath following or followers
+ * @param {*} page current pagination page
+ * @param {*} lst list we are building on
+ */
+const fetchAllWithPagination = async (apiPath, page, lst) => {
+    try {
+        const req = await queryGithubAPIRaw(`${apiPath}?page=${page}${API_PAGINATION}`);
+        if (req.body.hasOwnProperty("length")) {
+            const list = lst.concat(req.body);
+            if(page < API_MAX_PAGES && req.length === API_PAGINATION_SIZE) {
+                const redo = await fetchAllWithPagination(apiPath, page + 1, list);
+                return redo;
+            }
+            return list;
+        }
+    } catch (error) {
+        console.log("Error with api request");
+    }
+}
+
+
+/**
+ * Makes a copy of a JS object with certain properties
+ * 
+ * @param {*} props 
+ * @param {*} obj 
+ */
+const copyWithProperties = (props, obj) => {
+    let newO = new Object();
+    props.forEach(prop => {
+        newO[prop] = obj[prop];
+    })
+    return newO;
+}
+
+
+/**
+ * Combines the list of friends and followers ignoring duplicates
+ * that are already in the list. (person is both following and followed by someone)
+ * 
+ * This also removes any unused properties like events_url and organizations_url
+ * 
+ * @param {*} followingAndFollowers 
+ */
+const minimizeFriends = people => {
+    let friendLst = [];
+    let ids = new Set();
+    people.forEach(person => {
+        if(!ids.has(person.id)) {
+            ids.add(person.id);
+            friendLst.push({
+                login: person.login, 
+                avatar_url: person.avatar_url
+            });
+        }
+    });
+    return friendLst;
+}
+
+
+/**
+ * Fetches all the people that are either following or is followed
+ * by a person on github. This will cache the results to make simultaneous
+ * connections easier and less demanding on the github API.
+ * 
+ * @param {*} user 
+ */
+const queryFriends = async user => {
+    const cacheHit = cache.get("/friends/" + user);
+    if (cacheHit){
+        console.log("Friends cache hit");
+        return cacheHit;
+    }
+
+    try {
+        const followers = await fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWERS, 1, []);
+        const following = await fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWING, 1, []);
+        const fList = minimizeFriends(following.concat(followers));
+        cache.put(`/friends/${user}`, fList);
+        return fList;
+    } catch (error) {
+        console.log("API Error", err); 
+    }
+}
+
+
+/**
+ * 
+ * Fetches all of the members of an organization from the
+ * API or cache
+ *
+ * /orgs/RITlug/members?page=1
+ *
+ * @param {*} orgName 
+ */
+const getOrganizationMembers = async orgName => {
+    const cacheHit = cache.get("/org/users/" + orgName);
+    if (cacheHit){
+        console.log("Org members cache hit");
+        return cacheHit;
+    }
+
+    try {
+        const members = await fetchAllWithPagination(API_ORGS_PATH + orgName + "/members", 1, []);
+        const membersMin = minimizeFriends(members);
+        cache.put("/org/users/" + orgName, membersMin);
+        return membersMin;
+    } catch (error) {
+        console.log(error);
+    }
+}
+
+
+/**
+ * Minimizes the JSON for a list of repositories
+ * 
+ * @param {*} repositories 
+ */
+const minimizeRepositories = repositories => {
+    let rList = [];
+    repositories.forEach(repo => {
+        rList.push(copyWithProperties(["name", "created_at", "homepage", 
+            "description", "language", "forks", "watchers",
+            "open_issues_count", "license", "html_url"],
+            repo));
+    })
+
+    return rList;
+}
+
+
+/**
+ * Fetches all repositories from the API
+ * 
+ * @param {*} user name of org/user
+ * @param {*} orgsOrUsers  either /users/ or /orgs/
+ */
+const queryRepositories = async (user, orgsOrUsers) => {
+    const cacheHit = cache.get(user + REPOS_PATH);
+    if (cacheHit) {
+        console.log("Repositories cache hit");
+        return cacheHit;
+    }
+
+    try {
+        const repos = await fetchAllWithPagination(orgsOrUsers + user + REPOS_PATH, 1, []);
+        const minRepos = minimizeRepositories(repos);
+        cache.put(`${user}${REPOS_PATH}`, minRepos);
+        return minRepos;
+    } catch (error) {
+        console.log(error)
+        console.log("bad things went down");
+    }
+}
+
+
+/**
+ * /users/name/following/followers
+ */
+routes.get("/friends/:name", async (req, res)=> {
+    try {
+        const query = await queryFriends(req.params.name);
+        res.json(query);
+    } catch (error) {
+        res.status(500).json({error: 'API error fetching friends'});
+    }
+});
+
+
+routes.get("/org/users/:name", async (request, res) => {
+    try {
+        const orgMembers = await getOrganizationMembers(request.params.name);
+        res.json(orgMembers).end();
+    } catch (error) {
+        res.status(500).json({error: 'API error fetching friends'}).end();
+    }
+});
+
+
+routes.get("/repositories/:name", async (req, res) => {
+    try {
+        const repos = await queryRepositories(req.params.name, API_USER_PATH);
+        res.json(repos).end();
+    } catch (error) {
+        res.status(500).json({error: 'API error fetching friends'}).end();
+    }
+});
+
+
+routes.get("/org/repositories/:name", async (req, res) => {
+    try {
+        const repos = await queryRepositories(req.params.name, API_ORGS_PATH);
+        res.json(repos).end();
+    } catch (error) {
+        res.status(500).json({error: 'API error fetching friends'}).end();
+    }
+});
+
+
+routes.get('/*', (request, result) =>
+{
+    var gitHubAPIURL = request.url;
+
+    result.setHeader('Content-Type', 'application/json');
+    queryGitHubAPI(gitHubAPIURL).then((data)=>
+    {
+        if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id"))
+        {
+            result.write(JSON.stringify(data));
+        }
+        else
+        {
+            result.write("[]");
+        }
+        result.end();
+    }).catch((error)=>
+    {
+        try
+        {
+            if(error.hasOwnProperty("id") || error[0].hasOwnProperty("id"))
+            {
+                result.write(JSON.stringify(error));
+            }
+            else
+            {
+                result.write("[]");
+            }
+
+        }
+        catch(error) 
+        {
+            result.write("[]");
+        };
+        result.end();
+    });
+
+    if(cache.size() > 50000)
+    {
+        cache.clear();
+    }
+});
+
+module.exports = routes;

routes/index.js

@@ -1,6 +1,6 @@
 const routes = require('express').Router();
-const api = require('./api');
-routes.use('/api', api);
+const apiV1 = require('./api/v1');
+routes.use('/api', apiV1);
 
 routes.get("/", (request, response) => {
     response.redirect("index.html");