RanchiMall/utility-api
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

code refactoring

Browse Source
This commit is contained in:
sairaj mote 2023-12-07 22:33:33 +05:30
parent 5849877388
commit ef092d13b3
2 changed files with 21 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
index.js
View File

@ -8,40 +8,28 @@ const rateLimit = require('express-rate-limit');
const { parse: parseUrl } = require('url');
const { parse: parseHtml } = require('node-html-parser');
// Set up the allowed domains (replace with your specific domains)
const allowedDomains = process.env.ALLOWED_DOMAINS.split(',');
const app = express();
// pass the cors options to the cors middleware to enable CORS for the allowed domains
// const corsOptions = {
// origin: allowedDomains,
// optionsSuccessStatus: 200, // Some legacy browsers (IE11, various SmartTVs) choke on 204
// }
app.use(cors());
const port = process.env.PORT || 3000;
const host = process.env.HOST || 'localhost';
const host = process.env.HOST || '0.0.0.0';
// Middleware to parse JSON requests
app.use(express.json());
// Middleware to enable CORS
app.use(cors());
// Set up the allowed domains (replace with your specific domains)
const allowedDomains = process.env.ALLOWED_DOMAINS.split(',');
// Middleware to allow requests only from specified domains
// app.use((req, res, next) => {
// const { origin } = req.headers;
// // Check if the requesting origin is in the allowedDomains array
// if (allowedDomains.includes(origin)) {
// res.setHeader('Access-Control-Allow-Origin', origin);
// }
// // Other headers for handling preflight requests and allowing credentials if needed
// res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
// res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
// res.setHeader('Access-Control-Allow-Credentials', 'true');
// // Continue to the next middleware or route handler
// next();
// });
app.use(
rateLimit({
windowMs: 1 * 60 * 1000, // 1 minute
max: 10, // limit each IP request
max: 10, // limit each IP request per windowMs
})
);
@ -90,23 +78,22 @@ async function fetchAndHashContent(url, visitedUrls = new Set()) {
// API endpoint to start the recursive download and hashing
app.post('/hash', async (req, res) => {
try {
console.log('Request:', req.body);
let { url } = req.body;
if (!url) {
let { urls } = req.body;
if (!urls) {
return res.status(400).json({ error: 'Missing URL in the request parameters' });
}
if (!Array.isArray(url))
url = [url];
if (!Array.isArray(urls))
urls = [urls];
const promises = url.map(async (url) => {
const hashedContent = await fetchAndHashContent(url);
const promises = urls.map(async (urls) => {
const hashedContent = await fetchAndHashContent(urls);
const fileHash = await hashContent(Buffer.from(hashedContent, 'utf-8'));
return { url, fileHash };
return { urls, fileHash };
});
let results = await Promise.all(promises);
results = results.reduce((acc, { url, fileHash }) => {
acc[url] = fileHash;
results = results.reduce((acc, { urls, fileHash }) => {
acc[urls] = fileHash;
return acc;
}, {});

2
index.min.js vendored
View File

@ -1 +1 @@
require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),archiver=require("archiver"),rateLimit=require("express-rate-limit"),{parse:parseUrl}=require("url"),{parse:parseHtml}=require("node-html-parser"),app=express(),port=process.env.PORT||3e3,host=process.env.HOST||"localhost";app.use(express.json()),app.use(cors());const allowedDomains=process.env.ALLOWED_DOMAINS.split(",");async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer",timeout:1e4})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}async function downloadGitHubRepo(owner,repo){if(!owner||!repo)throw new Error("Missing owner or repo");const zipUrl=`https://github.com/${owner}/${repo}/archive/refs/heads/master.zip`;return(await axios.get(zipUrl,{responseType:"arraybuffer"})).data}app.use(rateLimit({windowMs:6e4,max:10})),app.get("/",((req,res)=>{res.send("Hello There!")})),app.post("/hash",(async(req,res)=>{try{console.log("Request:",req.body);let{url:url}=req.body;if(!url)return res.status(400).json({error:"Missing URL in the request parameters"});Array.isArray(url)||(url=[url]);const promises=url.map((async url=>{const hashedContent=await fetchAndHashContent(url);return{url:url,fileHash:await hashContent(Buffer.from(hashedContent,"utf-8"))}}));let results=await Promise.all(promises);results=results.reduce(((acc,{url:url,fileHash:fileHash})=>(acc[url]=fileHash,acc)),{}),res.json(results)}catch(error){console.error("Error:",error.message),res.status(500).json({error:"Internal Server Error"})}})),app.post("/download-repos",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing urls in the request parameters"});Array.isArray(urls)||(urls=[urls]);const archive=archiver("zip");res.attachment("repos.zip");const downloadPromises=urls.map((async url=>{const[owner,name]=url.split("/").slice(-2);if(!owner||!name)return void console.error(`Invalid url format: ${url}`);const zipBuffer=await downloadGitHubRepo(owner,name);archive.append(zipBuffer,{name:`${owner}-${name}.zip`})}));await Promise.all(downloadPromises),archive.finalize(),archive.pipe(res)}catch(error){console.error("Error:",error.message),res.status(500).json({error:"Internal Server Error"})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app;
require("dotenv").config();const express=require("express"),cors=require("cors"),axios=require("axios"),{createHash:createHash}=require("crypto"),archiver=require("archiver"),rateLimit=require("express-rate-limit"),{parse:parseUrl}=require("url"),{parse:parseHtml}=require("node-html-parser"),allowedDomains=process.env.ALLOWED_DOMAINS.split(","),app=express();app.use(cors());const port=process.env.PORT||3e3,host=process.env.HOST||"0.0.0.0";async function hashContent(content){const hash=createHash("sha256");return hash.update(content),hash.digest("hex")}async function fetchAndHashContent(url,visitedUrls=new Set){if(visitedUrls.has(url))return"";visitedUrls.add(url);const content=(await axios.get(url,{responseType:"arraybuffer",timeout:1e4})).data.toString("utf-8"),linkedResources=parseHtml(content).querySelectorAll('link[rel="stylesheet"], script[src]');return`${content}_${(await Promise.all(linkedResources.map((async resource=>{const resourceUrl=parseUrl(resource.getAttribute("href")||resource.getAttribute("src"),!0);let absoluteResourceUrl=resourceUrl.href;resourceUrl.hostname||(resourceUrl.path.startsWith("/")||url.endsWith("/")||(url+="/"),absoluteResourceUrl=`${url}${resourceUrl.path}`);const resourceContent=await fetchAndHashContent(absoluteResourceUrl,visitedUrls);return`${resourceUrl.path}_${resourceContent}`})))).join("_")}`}async function downloadGitHubRepo(owner,repo){if(!owner||!repo)throw new Error("Missing owner or repo");const zipUrl=`https://github.com/${owner}/${repo}/archive/refs/heads/master.zip`;return(await axios.get(zipUrl,{responseType:"arraybuffer"})).data}app.use(express.json()),app.use(rateLimit({windowMs:6e4,max:10})),app.get("/",((req,res)=>{res.send("Hello There!")})),app.post("/hash",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing URL in the request parameters"});Array.isArray(urls)||(urls=[urls]);const promises=urls.map((async urls=>{const hashedContent=await fetchAndHashContent(urls);return{urls:urls,fileHash:await hashContent(Buffer.from(hashedContent,"utf-8"))}}));let results=await Promise.all(promises);results=results.reduce(((acc,{urls:urls,fileHash:fileHash})=>(acc[urls]=fileHash,acc)),{}),res.json(results)}catch(error){console.error("Error:",error.message),res.status(500).json({error:"Internal Server Error"})}})),app.post("/download-repos",(async(req,res)=>{try{let{urls:urls}=req.body;if(!urls)return res.status(400).json({error:"Missing urls in the request parameters"});Array.isArray(urls)||(urls=[urls]);const archive=archiver("zip");res.attachment("repos.zip");const downloadPromises=urls.map((async url=>{const[owner,name]=url.split("/").slice(-2);if(!owner||!name)return void console.error(`Invalid url format: ${url}`);const zipBuffer=await downloadGitHubRepo(owner,name);archive.append(zipBuffer,{name:`${owner}-${name}.zip`})}));await Promise.all(downloadPromises),archive.finalize(),archive.pipe(res)}catch(error){console.error("Error:",error.message),res.status(500).json({error:"Internal Server Error"})}})),app.listen(port,host,(()=>{console.log(`Server is running at http://${host}:${port}`)})),module.exports=app;