From a65e99ea0529e24091e5ac8ec056aa43da56a75b Mon Sep 17 00:00:00 2001 From: sairaj mote Date: Mon, 8 Jan 2024 16:55:22 +0530 Subject: [PATCH] adding github hooks for improved performance --- index.js | 40 ++++++++++++++++++++++++++-------------- index.min.js | 2 +- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/index.js b/index.js index 16ccb56..1a01751 100644 --- a/index.js +++ b/index.js @@ -28,7 +28,7 @@ app.use(express.json()); app.use( rateLimit({ windowMs: 1 * 60 * 1000, // 1 minute - max: 10, // limit each IP request per windowMs + max: 20, // limit each IP request per windowMs }) ); @@ -107,32 +107,44 @@ app.post('/hash', async (req, res) => { // regex to identify owner and repo name from https://owner.github.io/repo-name const githubRepoRegex = /https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/; if (githubRepoRegex.test(urlWithoutHashAndQuery)) { - const [, owner, repo] = githubRepoRegex.exec(urlWithoutHashAndQuery) || [null, null, null,]; - const { data } = await axios.get(`https://api.github.com/repos/${owner}/${repo}`); - const lastUpdated = new Date(data.pushed_at); - - const cached = hashCache.get(urlWithoutHashAndQuery); - if (cached && cached.lastUpdated >= lastUpdated) { - hash = cached.hash; - } else { - const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery); - hash = await hashContent(Buffer.from(hashedContent, 'utf-8')); - hashCache.set(urlWithoutHashAndQuery, { hash, lastUpdated }); + if (!hashCache.has(urlWithoutHashAndQuery)) { + await fetchAndSaveAppHash(urlWithoutHashAndQuery) } + hash = hashCache.get(urlWithoutHashAndQuery).hash; } else { const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery); hash = await hashContent(Buffer.from(hashedContent, 'utf-8')); } - return { url, hash }; }); - let results = await Promise.all(promises); + const results = await Promise.all(promises); res.json(results); } catch (error) { res.status(500).json({ error: error.message }); } }); +async function fetchAndSaveAppHash(url, lastUpdated = Date.now()) { + const hashedContent = await fetchAndHashContent(url); + const hash = await hashContent(Buffer.from(hashedContent, 'utf-8')); + hashCache.set(url, { hash, lastUpdated }); +} + +app.post('/gitwh', async (req, res) => { + try { + // ignore if request is not from github + if (!req.headers['user-agent'].startsWith('GitHub-Hookshot/')) + return; + const { repository: { pushed_at, organization, name, has_pages } } = req.body; + if (!has_pages) + return; + const url = `https://${organization}.github.io/${name}` + await fetchAndSaveAppHash(url, pushed_at) + res.json({ message: 'success' }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}) // Start the server app.listen(port, host, () => { diff --git a/index.min.js b/index.min.js index c887b29..c34ca5d 100644 --- a/index.min.js +++ b/index.min.js @@ -1 +1 @@ -require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),rateLimit=require("express-rate-limit"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";function addProtocolToUrl(url){return url.startsWith("http://")||url.startsWith("https://")||(url="https://"+url),url}function parseUrlWithoutHashAndQuery(fullUrl){fullUrl=addProtocolToUrl(fullUrl);const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer"})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:10})),app.get("/",((req,res)=>{res.send("Hello There!")}));const hashCache=new Map;app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);let hash;const githubRepoRegex=/https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;if(githubRepoRegex.test(urlWithoutHashAndQuery)){const[,owner,repo]=githubRepoRegex.exec(urlWithoutHashAndQuery)||[null,null,null],{data:data}=await axios.get(`https://api.github.com/repos/${owner}/${repo}`),lastUpdated=new Date(data.pushed_at),cached=hashCache.get(urlWithoutHashAndQuery);if(cached&&cached.lastUpdated>=lastUpdated)hash=cached.hash;else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8")),hashCache.set(urlWithoutHashAndQuery,{hash:hash,lastUpdated:lastUpdated})}}else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8"))}return{url:url,hash:hash}}));let results=await Promise.all(promises);res.json(results)}catch(error){res.status(500).json({error:error.message})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app; \ No newline at end of file +require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),rateLimit=require("express-rate-limit"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";function addProtocolToUrl(url){return url.startsWith("http://")||url.startsWith("https://")||(url="https://"+url),url}function parseUrlWithoutHashAndQuery(fullUrl){fullUrl=addProtocolToUrl(fullUrl);const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer"})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:20})),app.get("/",((req,res)=>{res.send("Hello There!")}));const hashCache=new Map;async function fetchAndSaveAppHash(url,lastUpdated=Date.now()){const hashedContent=await fetchAndHashContent(url),hash=await hashContent(Buffer.from(hashedContent,"utf-8"));hashCache.set(url,{hash:hash,lastUpdated:lastUpdated})}app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);let hash;if(/https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/.test(urlWithoutHashAndQuery))hashCache.has(urlWithoutHashAndQuery)||await fetchAndSaveAppHash(urlWithoutHashAndQuery),hash=hashCache.get(urlWithoutHashAndQuery).hash;else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8"))}return{url:url,hash:hash}})),results=await Promise.all(promises);res.json(results)}catch(error){res.status(500).json({error:error.message})}})),app.post("/gitwh",(async(req,res)=>{try{if(!req.headers["user-agent"].startsWith("GitHub-Hookshot/"))return;const{repository:{pushed_at:pushed_at,organization:organization,name:name,has_pages:has_pages}}=req.body;if(!has_pages)return;const url=`https://${organization}.github.io/${name}`;await fetchAndSaveAppHash(url,pushed_at),res.json({message:"success"})}catch(err){res.status(500).json({error:err.message})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app; \ No newline at end of file