RanchiMall/utility-api
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/cicd

Browse Source
This commit is contained in:
Vivek Teega 2023-12-16 16:15:38 +05:30 committed by GitHub
commit b08fa1842c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 37 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
index.js
View File

@ -36,8 +36,15 @@ app.use(
app.get('/', (req, res) => {
res.send('Hello There!');
})
function addProtocolToUrl(url) {
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = 'https://' + url;
}
return url;
}
function parseUrlWithoutHashAndQuery(fullUrl) {
fullUrl = addProtocolToUrl(fullUrl);
const parsedUrl = new URL(fullUrl);
// Set the hash and search/query to empty strings
@ -63,15 +70,13 @@ async function fetchAndHashContent(url, visitedUrls = new Set()) {
}
visitedUrls.add(url);
const response = await axios.get(url, { responseType: 'arraybuffer', timeout: 10000 });
const content = response.data.toString('utf-8');
// Parse HTML content to identify linked resources
const root = parseHtml(content);
const linkedResources = root.querySelectorAll('link[rel="stylesheet"], script[src]');
// Fetch and hash linked resources
const linkedResourceHashes = await Promise.all(linkedResources.map(async (resource) => {
const linkedResource = await Promise.all(linkedResources.map(async (resource) => {
const resourceUrl = parseUrl(resource.getAttribute('href') || resource.getAttribute('src'), true);
let absoluteResourceUrl = resourceUrl.href;
if (!resourceUrl.hostname) {
@ -84,38 +89,50 @@ async function fetchAndHashContent(url, visitedUrls = new Set()) {
}));
// Combine the content and hashes of linked resources
return `${content}_${linkedResourceHashes.join('_')}`;
return `${content}_${linkedResource.join('_')}`;
}
const hashCache = new Map();
// API endpoint to start the recursive download and hashing
app.post('/hash', async (req, res) => {
try {
let { urls } = req.body;
if (!urls) {
return res.status(400).json({ error: 'Missing URL in the request parameters' });
return res.status(400).json({ error: 'Missing <urls> in the request parameters' });
}
if (!Array.isArray(urls))
urls = [urls];
const promises = urls.map(async (url) => {
const urlWithoutHashAndQuery = parseUrlWithoutHashAndQuery(url);
console.log(url, `Fetching and hashing ${urlWithoutHashAndQuery}`);
const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery);
const fileHash = await hashContent(Buffer.from(hashedContent, 'utf-8'));
return { urls, fileHash };
let hash;
// regex to identify owner and repo name from https://owner.github.io/repo-name
const githubRepoRegex = /https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;
if (githubRepoRegex.test(urlWithoutHashAndQuery)) {
const [, owner, repo] = githubRepoRegex.exec(urlWithoutHashAndQuery) || [null, null, null,];
const { data } = await axios.get(`https://api.github.com/repos/${owner}/${repo}`);
const lastUpdated = new Date(data.pushed_at);
const cached = hashCache.get(urlWithoutHashAndQuery);
if (cached && cached.lastUpdated >= lastUpdated) {
hash = cached.hash;
} else {
const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery);
hash = await hashContent(Buffer.from(hashedContent, 'utf-8'));
hashCache.set(urlWithoutHashAndQuery, { hash, lastUpdated });
}
} else {
const hashedContent = await fetchAndHashContent(urlWithoutHashAndQuery);
hash = await hashContent(Buffer.from(hashedContent, 'utf-8'));
}
return { url, hash };
});
let results = await Promise.all(promises);
results = results.reduce((acc, { urls, fileHash }) => {
acc[urls] = fileHash;
return acc;
}, {});
res.json(results);
} catch (error) {
console.error('Error:', error.message);
res.status(500).json({ error: 'Internal Server Error' });
res.status(500).json({ error: error.message });
}
});
@ -135,7 +152,7 @@ app.post('/download-repos', async (req, res) => {
let { urls } = req.body;
if (!urls) {
return res.status(400).json({ error: 'Missing urls in the request parameters' });
return res.status(400).json({ error: 'Missing <urls> in the request parameters' });
}
if (!Array.isArray(urls)) {
urls = [urls];
@ -167,8 +184,7 @@ app.post('/download-repos', async (req, res) => {
// Pipe the zip file to the response
archive.pipe(res);
} catch (error) {
console.error('Error:', error.message);
res.status(500).json({ error: 'Internal Server Error' });
res.status(500).json({ error: error.message });
}
});

2
index.min.js vendored
View File

@ -1 +1 @@
require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),archiver=require("archiver"),rateLimit=require("express-rate-limit"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";function parseUrlWithoutHashAndQuery(fullUrl){const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer",timeout:1e4})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}async function downloadGitHubRepo(owner,repo){if(!owner||!repo)throw new Error("Missing owner or repo");const zipUrl=`https://github.com/${owner}/${repo}/archive/refs/heads/master.zip`;return(await axios.get(zipUrl,{responseType:"arraybuffer"})).data}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:10})),app.get("/",((req,res)=>{res.send("Hello There!")})),app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing URL in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);console.log(url,`Fetching and hashing ${urlWithoutHashAndQuery}`);const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery),fileHash=await hashContent(Buffer.from(hashedContent,"utf-8"));return{urls:urls,fileHash:fileHash}}));let results=await Promise.all(promises);results=results.reduce(((acc,{urls:urls,fileHash:fileHash})=>(acc[urls]=fileHash,acc)),{}),res.json(results)}catch(error){console.error("Error:",error.message),res.status(500).json({error:"Internal Server Error"})}})),app.post("/download-repos",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing urls in the request parameters"});Array.isArray(urls)||(urls=[urls]);const archive=archiver("zip");res.attachment("repos.zip");const downloadPromises=urls.map((async url=>{const[owner,name]=url.split("/").slice(-2);if(!owner||!name)return void console.error(`Invalid url format: ${url}`);const zipBuffer=await downloadGitHubRepo(owner,name);archive.append(zipBuffer,{name:`${owner}-${name}.zip`})}));await Promise.all(downloadPromises),archive.finalize(),archive.pipe(res)}catch(error){console.error("Error:",error.message),res.status(500).json({error:"Internal Server Error"})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app;
require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),archiver=require("archiver"),rateLimit=require("express-rate-limit"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";function addProtocolToUrl(url){return url.startsWith("http://")||url.startsWith("https://")||(url="https://"+url),url}function parseUrlWithoutHashAndQuery(fullUrl){fullUrl=addProtocolToUrl(fullUrl);const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer",timeout:1e4})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:10})),app.get("/",((req,res)=>{res.send("Hello There!")}));const hashCache=new Map;async function downloadGitHubRepo(owner,repo){if(!owner||!repo)throw new Error("Missing owner or repo");const zipUrl=`https://github.com/${owner}/${repo}/archive/refs/heads/master.zip`;return(await axios.get(zipUrl,{responseType:"arraybuffer"})).data}app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing <urls> in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);let hash;const githubRepoRegex=/https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;if(githubRepoRegex.test(urlWithoutHashAndQuery)){const[,owner,repo]=githubRepoRegex.exec(urlWithoutHashAndQuery)||[null,null,null],{data:data}=await axios.get(`https://api.github.com/repos/${owner}/${repo}`),lastUpdated=new Date(data.pushed_at),cached=hashCache.get(urlWithoutHashAndQuery);if(cached&&cached.lastUpdated>=lastUpdated)hash=cached.hash;else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8")),hashCache.set(urlWithoutHashAndQuery,{hash:hash,lastUpdated:lastUpdated})}}else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8"))}return{url:url,hash:hash}}));let results=await Promise.all(promises);res.json(results)}catch(error){res.status(500).json({error:error.message})}})),app.post("/download-repos",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing <urls> in the request parameters"});Array.isArray(urls)||(urls=[urls]);const archive=archiver("zip");res.attachment("repos.zip");const downloadPromises=urls.map((async url=>{const[owner,name]=url.split("/").slice(-2);if(!owner||!name)return void console.error(`Invalid url format: ${url}`);const zipBuffer=await downloadGitHubRepo(owner,name);archive.append(zipBuffer,{name:`${owner}-${name}.zip`})}));await Promise.all(downloadPromises),archive.finalize(),archive.pipe(res)}catch(error){res.status(500).json({error:error.message})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app;