jrtechs-github-graphs/routes/api/v2.js

const routes = require('express').Router();
const got = require("got");
const cache = require('memory-cache');
const dotenv = require("dotenv").config();
const GITHUB_API = "https://api.github.com";
const authenticate = `client_id=${process.env.CLIENT_ID}&client_secret=${process.env.CLIENT_SECRET}`;

const API_FOLLOWING = "/following";
const API_FOLLOWERS = "/followers";
const API_USER_PATH = "/users/";
const API_ORGS_PATH = "/orgs/";
const API_PAGINATION_SIZE = 5; // 100 is the max, 30 is the default
// if this is too large, it would be infeasible to make graphs for people following popular people
const API_MAX_PAGES = 30;
const API_PAGINATION = "&per_page=" + API_PAGINATION_SIZE;

const REPOS_PATH = "/repos";


/**
 * Queries data from the github APi server and returns it as
 * a json object in a promise.
 * 
 * This makes **no attempt to cache**
 * 
 * @param {*} requestURL endpoint on githubapi: ex: /users/jrtechs/following
 */
const queryGithubAPIRaw = async requestURL => {
    let queryURL = requestURL.includes("?page=") ? `${GITHUB_API}${requestURL}&${authenticate}` :`${GITHUB_API}${requestURL}?${authenticate}`;
    console.log(queryURL);
    try
    {
        const req = await got(queryURL, { json: true });
        return req.body;
    }
    catch (error) 
    {
        console.log(error);
    }
}


/**
 * Queries data from the github api server
 * and caches the results locally.
 * 
 * @param {*} requestURL 
 */
const queryGitHubAPI = async requestURL => {
    const apiData = cache.get(requestURL);
    if (apiData) {
        console.log("Fetched From Cache");
        return apiData
    }

    try 
    {
        let d = await queryGithubAPIRaw(requestURL);
        return d;
        cache.put(requestURL, d);
    } catch (error) {
        console.log(error);
    }
}


/**
 * Fetches all content from a particular github api endpoint
 * using their pagination schema.
 * 
 * @param {*} username username of github client
 * @param {*} apiPath following or followers
 * @param {*} page current pagination page
 * @param {*} lst list we are building on
 */
const fetchAllWithPagination = async (apiPath, page, lst) => {
    try {
        const req = await queryGithubAPIRaw(`${apiPath}?page=${page}${API_PAGINATION}`);
        if (req.hasOwnProperty("length")) 
        {
            const list = lst.concat(req);
            console.log(req.length);
            //console.log(req.body);
            if(page < API_MAX_PAGES && req.length === API_PAGINATION_SIZE) {
                const redo = await fetchAllWithPagination(apiPath, page + 1, list);
                return redo;
            }
            return list;
        }
    } catch (error) {
        console.log("Error with api request");
    }
}


/**
 * Makes a copy of a JS object with certain properties
 * 
 * @param {*} props 
 * @param {*} obj 
 */
const copyWithProperties = (props, obj) => {
    let newO = new Object();
    props.forEach(prop => {
        newO[prop] = obj[prop];
    })
    return newO;
}


/**
 * Combines the list of friends and followers ignoring duplicates
 * that are already in the list. (person is both following and followed by someone)
 * 
 * This also removes any unused properties like events_url and organizations_url
 * 
 * @param {*} followingAndFollowers 
 */
const minimizeFriends = people => {
    let friendLst = [];
    let ids = new Set();
    people.forEach(person => {
        if(!ids.has(person.id)) {
            ids.add(person.id);
            friendLst.push({
                login: person.login, 
                avatar_url: person.avatar_url
            });
        }
    });
    return friendLst;
}


/**
 * Fetches all the people that are either following or is followed
 * by a person on github. This will cache the results to make simultaneous
 * connections easier and less demanding on the github API.
 * 
 * @param {*} user 
 */
const queryFriends = async user => {
    const cacheHit = cache.get("/friends/" + user);
    if (cacheHit){
        console.log("Friends cache hit");
        return cacheHit;
    }

    try {
        const followers = await fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWERS, 1, []);
        const following = await fetchAllWithPagination(API_USER_PATH + user + API_FOLLOWING, 1, []);
        const fList = minimizeFriends(following.concat(followers));
        cache.put(`/friends/${user}`, fList);
        return fList;
    } catch (error) {
        console.log("API Error", err); 
    }
}


/**
 * 
 * Fetches all of the members of an organization from the
 * API or cache
 *
 * /orgs/RITlug/members?page=1
 *
 * @param {*} orgName 
 */
const getOrganizationMembers = async orgName => {
    const cacheHit = cache.get("/org/users/" + orgName);
    if (cacheHit){
        console.log("Org members cache hit");
        return cacheHit;
    }

    try {
        const members = await fetchAllWithPagination(API_ORGS_PATH + orgName + "/members", 1, []);
        const membersMin = minimizeFriends(members);
        cache.put("/org/users/" + orgName, membersMin);
        return membersMin;
    } catch (error) {
        console.log(error);
    }
}


/**
 * Minimizes the JSON for a list of repositories
 * 
 * @param {*} repositories 
 */
const minimizeRepositories = repositories => {
    let rList = [];
    repositories.forEach(repo => {
        rList.push(copyWithProperties(["name", "created_at", "homepage", 
            "description", "language", "forks", "watchers",
            "open_issues_count", "license", "html_url"],
            repo));
    })

    return rList;
}


/**
 * Fetches all repositories from the API
 * 
 * @param {*} user name of org/user
 * @param {*} orgsOrUsers  either /users/ or /orgs/
 */
const queryRepositories = async (user, orgsOrUsers) => {
    const cacheHit = cache.get(user + REPOS_PATH);
    if (cacheHit) {
        console.log("Repositories cache hit");
        return cacheHit;
    }

    try {
        const repos = await fetchAllWithPagination(orgsOrUsers + user + REPOS_PATH, 1, []);
        const minRepos = minimizeRepositories(repos);
        cache.put(`${user}${REPOS_PATH}`, minRepos);
        return minRepos;
    } catch (error) {
        console.log(error)
        console.log("bad things went down");
    }
}

const EVENTS_PATH = "/events";


/**
 * 
 * @param {https://api.github.com/repos/jrtechs/bash_manager/events?page=1&per_page=100} repositoryName 
 */
const queryRepositoryEvents = async(user, repositoryName) =>
{
    console.log()
    try {
        const repo_events = await fetchAllWithPagination(`${REPOS_PATH}/${user}/${repositoryName}${EVENTS_PATH}`, 1, []);
        // const minRepos = minimizeRepositories(repos);
        // cache.put(`${user}${REPOS_PATH}`, minRepos);
        return repo_events;
    } catch (error) {
        console.log(error)
        console.log("bad things went down");
    }
}


/**
 * Minimizes the JSON for a list of commits
 * 
 * @param {*} commits 
 */
const minimizeCommits = commits => {
    let cList = [];
    commits.forEach(c => 
    {
        let obj = new Object();
        obj.message = c.commit.message;
        obj.date = c.commit.author.date;
        obj.sha = c.sha;
        if(c.author == null)
        {
            obj.name = c.commit.author.name;
        }
        else
        {
            obj.name = c.author.login;
        }
        cList.push(obj);
    });
    console.log("returning " + cList.length + " commits.")
    return cList;
}


const COMMITS_PATH = "/commits";

/**
 * TODO figure out if we should cache this because commits
 * frequently change opposed to friends
 * 
 * @param {https://api.github.com/repos/jrtechs/bash_manager/commits?page=1&per_page=100} repositoryName 
 */
const queryRepositoryCommits = async(user, repositoryName) =>
{
    try 
    {
        const path = `${REPOS_PATH}/${user}/${repositoryName}${COMMITS_PATH}`;

        const repo_commits = await fetchAllWithPagination(path, 1, []);
        const min_commits = minimizeCommits(repo_commits);
        // cache.put(`${user}${REPOS_PATH}`, minRepos);
        return min_commits;
    } catch (error) {
        console.log(error)
        console.log("error fetching commits");
    }
}


routes.get("/repositories/events/:name/:repository", async (req, res)=>
{
    try 
    {
        const query = await queryRepositoryEvents(req.params.name, req.params.repository);
        res.json(query).end();
    } catch (error) {
        res.status(500).json({error: 'API error fetching repository events'});
    }
});



routes.get("/repositories/commits/:name/:repository", async (req, res)=>
{
    try 
    {
        const query = await queryRepositoryCommits(req.params.name, req.params.repository);
        console.log("finished");
        res.json(query).end();
    } catch (error) {
        res.status(500).json({error: 'API error fetching repository events'});
    }
});


/**
 * /users/name/following/followers
 */
routes.get("/friends/:name", async (req, res)=> {
    try {
        const query = await queryFriends(req.params.name);
        res.json(query).end();
    } catch (error) {
        res.status(500).json({error: 'API error fetching friends'});
    }
});


routes.get("/org/users/:name", async (request, res) => {
    try {
        const orgMembers = await getOrganizationMembers(request.params.name);
        res.json(orgMembers).end();
    } catch (error) {
        res.status(500).json({error: 'API error fetching friends'}).end();
    }
});


routes.get("/repositories/:name", async (req, res) => {
    try {
        const repos = await queryRepositories(req.params.name, API_USER_PATH);
        res.json(repos).end();
    } catch (error) {
        res.status(500).json({error: 'API error fetching friends'}).end();
    }
});


routes.get("/org/repositories/:name", async (req, res) => {
    try {
        const repos = await queryRepositories(req.params.name, API_ORGS_PATH);
        res.json(repos).end();
    } catch (error) {
        res.status(500).json({error: 'API error fetching friends'}).end();
    }
});


routes.get('/*', (request, result) =>
{
    var gitHubAPIURL = request.url;

    result.setHeader('Content-Type', 'application/json');
    queryGitHubAPI(gitHubAPIURL).then((data)=>
    {
        if(data.hasOwnProperty("id") || data[0].hasOwnProperty("id"))
        {
            result.write(JSON.stringify(data));
        }
        else
        {
            result.write("[]");
        }
        result.end();
    }).catch((error)=>
    {
        try
        {
            if(error.hasOwnProperty("id") || error[0].hasOwnProperty("id"))
            {
                result.write(JSON.stringify(error));
            }
            else
            {
                result.write("[]");
            }

        }
        catch(error) 
        {
            result.write("[]");
        };
        result.end();
    });

    if(cache.size() > 50000)
    {
        cache.clear();
    }
});

module.exports = routes;