RanchiMall/utility-api
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update price-history.js adding CSV as backup

Browse Source 
Addign CSV as backup as fetch is not working
This commit is contained in:
tripathyr 2024-10-22 08:57:57 +05:30 committed by GitHub
parent 82d7d66927
commit 7c324c5852
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 83 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
routes/price-history.js
View File

@ -1,39 +1,91 @@
const express = require('express'); const express = require('express');
const router = express.Router(); const router = express.Router();
const cron = require('node-cron'); const cron = require('node-cron');
const fs = require('fs');
const path = require('path');
const csv = require('csv-parser'); // Assuming you have installed the csv-parser package
const PriceHistory = require('../models/price-history'); const PriceHistory = require('../models/price-history');
function loadHistoricToDb() { const CSV_FILE_PATH = '/home/production/deployed/utility-api/btc_price_history_full.csv';
const now = parseInt(Date.now() / 1000);
Promise.all([ // Function to read CSV file and return data
fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-USD?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res) => res.text()), function readCsvFile() {
fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-INR?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res) => res.text()), return new Promise((resolve, reject) => {
]) const results = [];
.then(async ([usd, inr]) => { fs.createReadStream(CSV_FILE_PATH)
const usdData = usd.split("\n").slice(1); .pipe(csv())
const inrData = inr.split("\n").slice(1); .on('data', (data) => results.push(data))
const priceHistoryData = []; .on('end', () => resolve(results))
for (let i = 0; i < usdData.length; i++) { .on('error', (err) => reject(err));
const [date, open, high, low, close, adjClose, volume] = usdData[i].split(","); });
const [date2, open2, high2, low2, close2, adjClose2, volume2] = inrData[i].split(",");
priceHistoryData.push({
date: new Date(date).getTime(),
asset: "btc",
usd: parseFloat(parseFloat(close).toFixed(2)),
inr: parseFloat(parseFloat(close2).toFixed(2)),
});
}
// update many
await PriceHistory.deleteMany({ asset: 'btc' });
await PriceHistory.insertMany(priceHistoryData);
})
.catch((err) => {
console.log(err);
})
} }
// Function to load historic data into the database without deleting old data
async function loadHistoricToDb() {
const now = parseInt(Date.now() / 1000);
try {
const [usd, inr] = await Promise.all([
fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-USD?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res) => res.text()),
fetch(`https://query1.finance.yahoo.com/v7/finance/download/BTC-INR?period1=1410912000&period2=${now}&interval=1d&events=history&includeAdjustedClose=true`).then((res) => res.text())
]);
// If fetch succeeds, process the fetched data
const usdData = usd.split("\n").slice(1);
const inrData = inr.split("\n").slice(1);
const operations = [];
for (let i = 0; i < usdData.length; i++) {
const [date, open, high, low, close, adjClose, volume] = usdData[i].split(",");
const [date2, open2, high2, low2, close2, adjClose2, volume2] = inrData[i].split(",");
operations.push({
updateOne: {
filter: { date: new Date(date).getTime(), asset: "btc" },
update: {
$set: {
usd: parseFloat(parseFloat(close).toFixed(2)),
inr: parseFloat(parseFloat(close2).toFixed(2)),
}
},
upsert: true
}
});
}
// Perform bulk upsert operations
await PriceHistory.bulkWrite(operations);
console.log("Data upserted successfully from API.");
} catch (fetchError) {
// If fetch fails, read from the CSV file
console.error("Failed to fetch data. Falling back to CSV file:", fetchError);
try {
const csvData = await readCsvFile();
const operations = csvData.map((row) => ({
updateOne: {
filter: { date: new Date(row.date).getTime(), asset: "btc" },
update: {
$set: {
usd: parseFloat(row.usd),
inr: parseFloat(row.inr),
}
},
upsert: true
}
}));
// Perform bulk upsert operations
await PriceHistory.bulkWrite(operations);
console.log("Data upserted successfully from CSV.");
} catch (csvError) {
console.error("Error reading CSV file:", csvError);
}
}
}
loadHistoricToDb(); loadHistoricToDb();
// Route to handle price history requests
router.get("/", async (req, res) => { router.get("/", async (req, res) => {
console.log('price-history'); console.log('price-history');
try { try {
@ -93,36 +145,9 @@ router.get("/", async (req, res) => {
} }
}) })
// Cron job to periodically update the price history
cron.schedule('0 */4 * * *', async () => { cron.schedule('0 */4 * * *', async () => {
try { await loadHistoricToDb();
// will return a csv file });
const [usd, inr] = await Promise.all([
fetch("https://query1.finance.yahoo.com/v7/finance/download/BTC-USD").
then((res) => res.text()),
fetch("https://query1.finance.yahoo.com/v7/finance/download/BTC-INR").
then((res) => res.text())
]);
const usdData = usd.split("\n").slice(1);
const inrData = inr.split("\n").slice(1);
for (let i = 0; i < usdData.length; i++) {
const [date, open, high, low, close, adjClose, volume] = usdData[i].split(",");
const [date2, open2, high2, low2, close2, adjClose2, volume2] = inrData[i].split(",");
const priceHistoryData = {
date: new Date(date).getTime(),
asset: "btc",
usd: parseFloat(parseFloat(close).toFixed(2)),
inr: parseFloat(parseFloat(close2).toFixed(2)),
};
await PriceHistory.findOneAndUpdate(
{ date: priceHistoryData.date, asset: priceHistoryData.asset },
priceHistoryData,
{ upsert: true }
);
}
} catch (err) {
console.log(err);
}
})
module.exports = router; module.exports = router;