Refactored queryFriends()

routes/api/v2.js

@@ -0,0 +1,355 @@
+const routes = require('express').Router();
+const got = require("got");
+const cache = require('memory-cache');
+const dotenv = require("dotenv").config();
+const GITHUB_API = "https://api.github.com";
+const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`;
+
+const API_FOLLOWING = "/following";
+const API_FOLLOWERS = "/followers";
+const API_USER_PATH = "/users/";
+const API_ORGS_PATH = "/orgs/";
+const API_PAGINATION_SIZE = 100; // 100 is the max, 30 is the default
+// if this is too large, it would be infeasible to make graphs for people following popular people
+const API_MAX_PAGES = 2;
+const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;
+
+const REPOS_PATH = "/repos";
+
+
+/**
+ * Queries data from the github APi server and returns it as
+ * a json object in a promise.
+ * 
+ * This makes no attempt to cache
+ * 
+ * @param {*} requestURL endpoint on githubapi: ex: /users/jrtechs/following
+ */
+const queryGithubAPIRaw = async requestURL => {
+    let queryURL = requestURL.includes("?page=") 
+        ? `${GITHUB_API}${requestURL}&${authenticate}` 
+        :`${GITHUB_API}${requestURL}?${authenticate}`;
+    console.log(queryURL);
+    try {
+        const req = await got(queryURL, { json: true });
+        cache.put(requestURL, req);
+        return req;
+    } catch (error) {
+        console.log(error);
+        cache.put(requestURL, `${error.statusCode} - ${error.statusMessage}`);
+    }
+}
+
+
+/**
+ * Queries data from the github api server
+ * and caches the results locally.
+ * 
+ * @param {*} requestURL 
+ */
+const queryGitHubAPI = async requestURL => {
+    const apiData = cache.get(requestURL);
+
+    if (apiData) {
+        console.log("Fetched From Cache");
+        return apiData
+    }
+
+    try {
+        return await queryGithubAPIRaw(requestURL);
+    } catch (error) {
+        console.log(error);
+    }
+}
+
+
+/**
+ * Fetches all content from a particular github api endpoint
+ * using their pagination schema.
+ * 
+ * @param {*} username username of github client
+ * @param {*} apiPath following or followers
+ * @param {*} page current pagination page
+ * @param {*} lst list we are building on
+ */
+const fetchAllWithPagination = async (apiPath, page, lst) => {
+    try {
+        const req = await queryGithubAPIRaw(`${apiPath}?page=${page}${API_PAGINATION}`);
+        if (req) {
+            const list = lst.concat(req.body);
+            if(page < API_MAX_PAGES && req.length === API_PAGINATION_SIZE) {
+                const redo = await fetchAllWithPagination(apiPath, page + 1, list);
+                return redo;
+            }
+            return list;
+        }
+    } catch (error) {
+        console.log("Error with api request");
+    }
+}
+
+
+/**
+ * Makes a copy of a JS object with certain properties
+ * 
+ * @param {*} props 
+ * @param {*} obj 
+ */
+function copyWithProperties(props, obj)
+{
+    var newO = new Object();
+    for(var i =0; i < props.length; i++)
+    {
+        newO[props[i]] = obj[props[i]];
+    }
+    return newO;
+}
+
+
+/**
+ * Combines the list of friends and followers ignoring duplicates
+ * that are already in the list. (person is both following and followed by someone)
+ * 
+ * This also removes any unused properties like events_url and organizations_url
+ * 
+ * @param {*} followingAndFollowers 
+ */
+function minimizeFriends(people)
+{
+    var friendLst = [];
+
+    var ids = new Set();
+
+    for(var i = 0; i < people.length; i++)
+    {
+        if(!ids.has(people[i].id))
+        {
+            ids.add(people[i].id);
+            friendLst.push({
+                login: people[i].login, 
+                avatar_url: people[i].avatar_url
+            });
+        }
+    }
+    return friendLst;
+}
+
+
+/**
+ * Fetches all the people that are either following or is followed
+ * by a person on github. This will cache the results to make simultaneous
+ * connections easier and less demanding on the github API.
+ * 
+ * @param {*} user 
+ */
+const queryFriends = async user => {
+    const cacheHit = cache.get("/friends/" + user);
+    if (cacheHit){
+        console.log("Friends cache hit");
+        return cacheHit;
+    }
+
+    try {
+        const followers = await fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWERS, 1, []);
+        const following = await fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWING, 1, []);
+        const fList = minimizeFriends(following.concat(followers));
+        cache.put(`/friends/${user}`, fList);
+        return fList;
+    } catch (error) {
+        console.log("API Error", err); 
+    }
+}
+
+
+/**
+ * 
+ * Fetches all of the members of an organization from the
+ * API or cache
+ *
+ * /orgs/RITlug/members?page=1
+ *
+ * @param {*} orgName 
+ */
+function getOrganizationMembers(orgName)
+{
+    const cacheHit = cache.get("/org/users/" + orgName);
+    return new Promise((resolve, reject)=>
+    {
+        if(cacheHit == null)
+        {
+            fetchAllWithPagination(API_ORGS_PATH + orgName + "/members", 1, []).then((mems)=>
+            {
+                var minimized = minimizeFriends(mems);
+                resolve(minimized);
+                cache.put("/org/users/" + orgName, minimized);
+            }).catch((err)=>
+            {
+                console.log(err)
+            })
+        }
+        else
+        {
+            console.log("Org members cache hit");
+            resolve(cacheHit);
+        }
+    });
+}
+
+
+/**
+ * Minimizes the JSON for a list of repositories
+ * 
+ * @param {*} repositories 
+ */
+function minimizeRepositories(repositories)
+{
+    var rList = [];
+
+    for(var i = 0; i < repositories.length; i++)
+    {
+        rList.push(copyWithProperties(["name", "created_at", "homepage", 
+            "description", "language", "forks", "watchers",
+            "open_issues_count", "license", "html_url"],
+            repositories[i]));
+    }
+    return rList;
+}
+
+
+/**
+ * Fetches all repositories from the API
+ * 
+ * @param {*} user name of org/user
+ * @param {*} orgsOrUsers  either /users/ or /orgs/
+ */
+function queryRepositories(user, orgsOrUsers)
+{
+    const cacheHit = cache.get(user + REPOS_PATH);
+    return new Promise((resolve, reject)=>
+    {
+        if(cacheHit == null)
+        {
+            fetchAllWithPagination(orgsOrUsers + user + REPOS_PATH, 1, []).then((repos)=>
+            {
+                var minimized = minimizeRepositories(repos);
+                resolve(minimized);
+                cache.put(user + REPOS_PATH, minimized);
+            }).catch((err)=>
+            {
+                console.log(err)
+                console.log("bad things went down");
+            })
+        }
+        else
+        {
+            console.log("Repositories cache hit");
+            resolve(cacheHit);
+        }
+    });
+}
+
+
+/**
+ * /users/name/following/followers
+ */
+routes.get("/friends/:name", async (req, res)=> {
+    try {
+        const query = await queryFriends(req.params.name);
+        res.json(query);
+    } catch (error) {
+        res.status(500).json({error: 'API error fetching friends'});
+    }
+});
+
+
+
+routes.get("/org/users/:name", (request, result)=>
+{
+    getOrganizationMembers(request.params.name).then(friends=>
+        {
+            result.json(friends)
+                .end();
+        }).catch(error=>
+        {
+            result.status(500)
+                .json({error: 'API error fetching friends'})
+                .end();
+        });
+});
+
+
+
+routes.get("/repositories/:name", (request, result)=>
+{
+    queryRepositories(request.params.name, API_USER_PATH).then(repos=>
+        {
+            result.json(repos)
+                .end();
+        }).catch(error=>
+        {
+            result.status(500)
+                .json({error: 'API error fetching friends'})
+                .end();
+        });
+});
+
+
+routes.get("/org/repositories/:name", (request, result)=>
+{
+    queryRepositories(request.params.name, API_ORGS_PATH).then(repos=>
+        {
+            result.json(repos)
+                .end();
+        }).catch(error=>
+        {
+            result.status(500)
+                .json({error: 'API error fetching friends'})
+                .end();
+        });
+});
+
+
+routes.get('/*', (request, result) =>
+{
+    var gitHubAPIURL = request.url;
+
+    result.setHeader('Content-Type', 'application/json');
+    queryGitHubAPI(gitHubAPIURL).then((data)=>
+    {
+        if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id"))
+        {
+            result.write(JSON.stringify(data));
+        }
+        else
+        {
+            result.write("[]");
+        }
+        result.end();
+    }).catch((error)=>
+    {
+        try
+        {
+            if(error.hasOwnProperty("id") || error[0].hasOwnProperty("id"))
+            {
+                result.write(JSON.stringify(error));
+            }
+            else
+            {
+                result.write("[]");
+            }
+
+        }
+        catch(error) 
+        {
+            result.write("[]");
+        };
+        result.end();
+    });
+
+    if(cache.size() > 50000)
+    {
+        cache.clear();
+    }
+});
+
+module.exports = routes;