diff --git a/index.js b/index.js index af6444e..16ccb56 100644 --- a/index.js +++ b/index.js @@ -3,7 +3,6 @@ const express = require('express'); const cors = require('cors'); const axios = require('axios'); const { createHash } = require('crypto'); -const archiver = require('archiver'); const rateLimit = require('express-rate-limit'); const { parse: parseUrl, URL } = require('url'); const { parse: parseHtml } = require('node-html-parser'); @@ -70,7 +69,7 @@ async function fetchAndHashContent(url, visitedUrls = new Set()) { } visitedUrls.add(url); - const response = await axios.get(url, { responseType: 'arraybuffer', timeout: 10000 }); + const response = await axios.get(url, { responseType: 'arraybuffer' }); const content = response.data.toString('utf-8'); // Parse HTML content to identify linked resources const root = parseHtml(content); @@ -102,7 +101,6 @@ app.post('/hash', async (req, res) => { } if (!Array.isArray(urls)) urls = [urls]; - const promises = urls.map(async (url) => { const urlWithoutHashAndQuery = parseUrlWithoutHashAndQuery(url); let hash; @@ -136,58 +134,6 @@ app.post('/hash', async (req, res) => { } }); -// Function to download a GitHub repo as a zip file -async function downloadGitHubRepo(owner, repo) { - if (!owner || !repo) { - throw new Error('Missing owner or repo'); - } - const zipUrl = `https://github.com/${owner}/${repo}/archive/refs/heads/master.zip`; - const response = await axios.get(zipUrl, { responseType: 'arraybuffer' }); - return response.data; -} - -// Endpoint to download and zip GitHub repositories -app.post('/download-repos', async (req, res) => { - try { - let { urls } = req.body; - - if (!urls) { - return res.status(400).json({ error: 'Missing in the request parameters' }); - } - if (!Array.isArray(urls)) { - urls = [urls]; - } - - const archive = archiver('zip'); - res.attachment('repos.zip'); - - // Create an array of promises for each repository download - const downloadPromises = urls.map(async (url) => { - const [owner, name] = url.split('/').slice(-2); - - if (!owner || !name) { - console.error(`Invalid url format: ${url}`); - return; - } - - const zipBuffer = await downloadGitHubRepo(owner, name); - // Add the zip file to the archiver - archive.append(zipBuffer, { name: `${owner}-${name}.zip` }); - }); - - // Wait for all promises to complete - await Promise.all(downloadPromises); - - // Finalize the zip file - archive.finalize(); - - // Pipe the zip file to the response - archive.pipe(res); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - // Start the server app.listen(port, host, () => { console.log(`Server is running at http://${host}:${port}`); diff --git a/index.min.js b/index.min.js index ac91f98..c887b29 100644 --- a/index.min.js +++ b/index.min.js @@ -1 +1 @@ -require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),archiver=require("archiver"),rateLimit=require("express-rate-limit"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";function addProtocolToUrl(url){return url.startsWith("http://")||url.startsWith("https://")||(url="https://"+url),url}function parseUrlWithoutHashAndQuery(fullUrl){fullUrl=addProtocolToUrl(fullUrl);const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer",timeout:1e4})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:10})),app.get("/",((req,res)=>{res.send("Hello There!")}));const hashCache=new Map;async function downloadGitHubRepo(owner,repo){if(!owner||!repo)throw new Error("Missing owner or repo");const zipUrl=`https://github.com/${owner}/${repo}/archive/refs/heads/master.zip`;return(await axios.get(zipUrl,{responseType:"arraybuffer"})).data}app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);let hash;const githubRepoRegex=/https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;if(githubRepoRegex.test(urlWithoutHashAndQuery)){const[,owner,repo]=githubRepoRegex.exec(urlWithoutHashAndQuery)||[null,null,null],{data:data}=await axios.get(`https://api.github.com/repos/${owner}/${repo}`),lastUpdated=new Date(data.pushed_at),cached=hashCache.get(urlWithoutHashAndQuery);if(cached&&cached.lastUpdated>=lastUpdated)hash=cached.hash;else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8")),hashCache.set(urlWithoutHashAndQuery,{hash:hash,lastUpdated:lastUpdated})}}else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8"))}return{url:url,hash:hash}}));let results=await Promise.all(promises);res.json(results)}catch(error){res.status(500).json({error:error.message})}})),app.post("/download-repos",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing in the request parameters"});Array.isArray(urls)||(urls=[urls]);const archive=archiver("zip");res.attachment("repos.zip");const downloadPromises=urls.map((async url=>{const[owner,name]=url.split("/").slice(-2);if(!owner||!name)return void console.error(`Invalid url format: ${url}`);const zipBuffer=await downloadGitHubRepo(owner,name);archive.append(zipBuffer,{name:`${owner}-${name}.zip`})}));await Promise.all(downloadPromises),archive.finalize(),archive.pipe(res)}catch(error){res.status(500).json({error:error.message})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app; \ No newline at end of file +require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),rateLimit=require("express-rate-limit"),{parse:parseUrl,URL:URL}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";function addProtocolToUrl(url){return url.startsWith("http://")||url.startsWith("https://")||(url="https://"+url),url}function parseUrlWithoutHashAndQuery(fullUrl){fullUrl=addProtocolToUrl(fullUrl);const parsedUrl=new URL(fullUrl);parsedUrl.hash="",parsedUrl.search="";return parsedUrl.toString()}async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer"})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:10})),app.get("/",((req,res)=>{res.send("Hello There!")}));const hashCache=new Map;app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async url=>{const urlWithoutHashAndQuery=parseUrlWithoutHashAndQuery(url);let hash;const githubRepoRegex=/https?:\/\/([\w-]+)\.github\.io\/([\w-]+)/;if(githubRepoRegex.test(urlWithoutHashAndQuery)){const[,owner,repo]=githubRepoRegex.exec(urlWithoutHashAndQuery)||[null,null,null],{data:data}=await axios.get(`https://api.github.com/repos/${owner}/${repo}`),lastUpdated=new Date(data.pushed_at),cached=hashCache.get(urlWithoutHashAndQuery);if(cached&&cached.lastUpdated>=lastUpdated)hash=cached.hash;else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8")),hashCache.set(urlWithoutHashAndQuery,{hash:hash,lastUpdated:lastUpdated})}}else{const hashedContent=await fetchAndHashContent(urlWithoutHashAndQuery);hash=await hashContent(Buffer.from(hashedContent,"utf-8"))}return{url:url,hash:hash}}));let results=await Promise.all(promises);res.json(results)}catch(error){res.status(500).json({error:error.message})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app; \ No newline at end of file