You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
287 lines
9.7 KiB
287 lines
9.7 KiB
const mongoose = require('mongoose');
|
|
const mongoURI = process.env.DATABASE_URL;
|
|
mongoose.connect(mongoURI);
|
|
mongoose.connection.on('connected', ()=>{console.log('MongoDB - Worker - Connected')})
|
|
.on('disconnected', ()=>{console.log('MongoDB - Disconnect')})
|
|
.on('error', (error)=>console.log('Mongoose Error:', error));
|
|
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const { Stream } = require('stream');
|
|
const {Manga, Chapter} = require('./lib');
|
|
const FavModel = require('./models/favorite')
|
|
const EventEmitter = require('events');
|
|
const Emitter = new EventEmitter();
|
|
let config;
|
|
if(fs.existsSync('./config.json'))
|
|
{
|
|
config = require('./config.json')
|
|
}
|
|
const BaseDir = config?.saveFolder || './public/Saved';
|
|
|
|
var Queue = [];
|
|
var slots = 2;
|
|
|
|
var shouldKeepOnDownloading = true;
|
|
|
|
// updateQueue();
|
|
// var queueUpdateTimer = setInterval(() => {
|
|
// updateQueue();
|
|
// }, 5*60*1000);
|
|
|
|
|
|
// process.on('message', m=>
|
|
// {
|
|
// if(m.action==='add')
|
|
// {
|
|
// clearInterval(queueUpdateTimer);
|
|
// updateQueue();
|
|
// queueUpdateTimer = setInterval(() => {
|
|
// updateQueue();
|
|
// }, 5*60*1000);
|
|
// return
|
|
// }
|
|
|
|
// if(m.action==='remove') return removeFromQueue(m.id)
|
|
// if(m.action==='pause') return shouldKeepOnDownloading=false;
|
|
// })
|
|
|
|
function removeFromQueue(id)
|
|
{
|
|
const ItemDownloading = Queue.filter(manga => manga.inProgress == true)[0];
|
|
if(ItemDownloading) shouldKeepOnDownloading = false;
|
|
Queue = Queue.filter(manga => manga._id !== id);
|
|
|
|
}
|
|
Emitter.on('queueUpdate', ()=>
|
|
{
|
|
if(!Queue[0]) return
|
|
if(!Queue[0].inProgress)
|
|
{
|
|
Queue[0].inProgress = true;
|
|
download(Queue[0])
|
|
}
|
|
} );
|
|
|
|
async function updateQueue()
|
|
{
|
|
shouldKeepOnDownloading = true;
|
|
const Scanlators = fs.readdirSync(BaseDir);
|
|
const Favs = await FavModel.find();
|
|
let Mangas = [];
|
|
for(let i =0; i<Scanlators.length; i++)
|
|
{
|
|
let mangas = fs.readdirSync(path.join(BaseDir, Scanlators[i]));
|
|
if(mangas.length < 0) continue
|
|
for(var j = 0; j<mangas.length; j++)
|
|
{
|
|
let chapters = fs.readdirSync(path.join(BaseDir, Scanlators[i], mangas[j]));
|
|
if(chapters.length==0) continue
|
|
chapters.sort((a, b) => { return parseInt(a.split('_')[1]) - parseInt(b.split('_')[1]) });
|
|
let latestDownloaded = parseInt(chapters[chapters.length-1].split('_')[1]);
|
|
let manga = {scanlator:Scanlators[i], title:mangas[j], latestDownloaded}
|
|
Mangas.push(manga)
|
|
}
|
|
}
|
|
let FavedAndDownloaded = [];
|
|
let FavedButNotDownloaded = null;
|
|
for (let i = 0; i < Mangas.length; i++)
|
|
{
|
|
FavedAndDownloaded = FavedAndDownloaded.concat(Favs.filter(item =>
|
|
{
|
|
return (item.scanlator === Mangas[i].scanlator && item.title === Mangas[i].title);
|
|
}).map((matchedItem) =>{
|
|
return ({
|
|
...matchedItem._doc,
|
|
latestDownloaded: Mangas[i].latestDownloaded
|
|
})
|
|
})
|
|
);
|
|
}
|
|
FavedButNotDownloaded = Favs.filter(fav =>
|
|
{
|
|
// return !FavedAndDownloaded.includes(fav)
|
|
return !FavedAndDownloaded.some(fad => fad._id === fav._id);
|
|
});
|
|
if(FavedButNotDownloaded)
|
|
{
|
|
for(let i = 0; i < FavedButNotDownloaded.length; i++)
|
|
{
|
|
(async(i)=>
|
|
{
|
|
const {title, link, scanlator, _id} = FavedButNotDownloaded[i];
|
|
const manga = await new Manga(scanlator, link, title).get();
|
|
let id = _id.toString();
|
|
addToQueue(manga, id);
|
|
})(i)
|
|
}
|
|
}
|
|
if(FavedAndDownloaded)
|
|
{
|
|
for(let i = 0; i<FavedAndDownloaded.length; i++)
|
|
{
|
|
(async(i)=>
|
|
{
|
|
const {title, link, scanlator, latestDownloaded, _id} = FavedAndDownloaded[i];
|
|
const manga = await new Manga(scanlator, link, title).get();
|
|
let id = _id.toString();
|
|
checkForMissingFiles(manga)
|
|
if(latestDownloaded<manga.latestChap) addToQueue(manga, id, latestDownloaded);
|
|
})(i);
|
|
}
|
|
}
|
|
function addToQueue(manga, _id,startFrom=0)
|
|
{
|
|
const List = manga.List.filter(item => item.num > startFrom);
|
|
Queue.push({...manga, _id, List, startFrom})
|
|
Emitter.emit('queueUpdate', '');
|
|
}
|
|
}
|
|
async function checkForMissingFiles(manga)
|
|
{
|
|
const {scanlator, title, List} = manga;
|
|
let auxList = List.sort((a,b)=>{return a.num - b.num})
|
|
for(var i = 0; i<auxList.length; i++)
|
|
{
|
|
const {link} = auxList[i];
|
|
const destPath = path.join(BaseDir, scanlator, title, 'CH_'+(i+1));
|
|
const chapterDownloaded = fs.existsSync(destPath);
|
|
if(!chapterDownloaded)
|
|
{
|
|
if(shouldKeepOnDownloading)
|
|
{
|
|
const chapter = await new Chapter(scanlator, link, title, i+1).get();
|
|
await downloadChapter(chapter);
|
|
}
|
|
continue;
|
|
}
|
|
const chapterDir = fs.readdirSync(destPath);
|
|
const chapter = await new Chapter(scanlator, link, title, i+1).get();
|
|
let auxChapterDir = chapterDir.map(item=>{return item.split('.')[0]})
|
|
if(!chapter?.List) continue;
|
|
var auxChapterList = chapter.List.map(item=>{return item.split('/')[item.split('/').length-1].split('.')[0]})
|
|
var incompleteImages = [];
|
|
for(var k = 0; k<auxChapterDir.length; k++)
|
|
{
|
|
const auxDir = chapterDir.sort();
|
|
const _path = path.join(destPath, auxDir[k]);
|
|
const isComplete = await isImageComplete(_path);
|
|
if(!isComplete) incompleteImages.push(auxDir[k].split('.')[0])
|
|
}
|
|
const missingImages =[...incompleteImages, ...auxChapterList.filter(item=>!auxChapterDir.includes(item))];
|
|
if(missingImages.length == 0) continue;
|
|
for(var j = 0; j<missingImages.length; j++)
|
|
{
|
|
const imageLink = chapter.List.filter(item=>item.split('/')[item.split('/').length-1].split('.')[0] == missingImages[j])[0]
|
|
const destination = `${destPath}/${imageLink.split('/')[(imageLink.split('/').length - 1)]}`
|
|
if(shouldKeepOnDownloading)
|
|
{
|
|
const download = await downloadImage(imageLink,destination, 5);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
*
|
|
* @param {Path} filePath
|
|
* @returns {Boolean}
|
|
*/
|
|
async function isImageComplete(filePath)
|
|
{
|
|
async function IsFileComplete(filePath) {
|
|
try {
|
|
await sharp(filePath, { sequentialRead: true })
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
return true
|
|
} catch (err) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
async function download(manga)
|
|
{
|
|
if(slots <=0) return
|
|
slots--;
|
|
const { scanlator, title} = manga;
|
|
const List = manga.List.sort((a, b) => { return a.num - b.num });
|
|
for (let i = 0; i < List.length; i++)
|
|
{
|
|
const ch = List[i];
|
|
if (isNaN(ch.num)) continue;
|
|
const { num, link } = ch;
|
|
const chapter = await new Chapter(scanlator, link, title, num).get();
|
|
// console.log(chapter)
|
|
if (!chapter?.List?.length) continue;
|
|
const dlChapter = await downloadChapter(chapter);
|
|
if(dlChapter=='UserStoped') break;
|
|
}
|
|
Queue.shift();
|
|
slots++;
|
|
Emitter.emit('queueUpdate', '');
|
|
}
|
|
async function downloadChapter(chapter)
|
|
{
|
|
const {List, scanlator, title, chNum } = chapter;
|
|
const images = List;
|
|
for (let i = 0; i < images.length; i++) {
|
|
const img = images[i];
|
|
const destination = `./public/Saved/${scanlator}/${title}/CH_${chNum}/${img.split('/')[(img.split('/').length - 1)]}`;
|
|
try {
|
|
const downloaded = await downloadImage(img, destination);
|
|
console.log(`Downloaded: Scanlator:${scanlator}; Manga: ${title} Chapter:${chNum}; IMG:${img.split('/')[(img.split('/').length - 1)]} ${downloaded.status}`);
|
|
} catch (error) {
|
|
console.error(`Error downloading: ${img}: ${error}`);
|
|
if(!shouldKeepOnDownloading)
|
|
{
|
|
ReturnError = error.Error;
|
|
break;
|
|
};
|
|
}
|
|
}
|
|
return !shouldKeepOnDownloading?'UserStoped':{chNum};
|
|
}
|
|
/**
|
|
*
|
|
* @param {Link} url
|
|
* @param {Path} destPath
|
|
* @param {Number} maxRetries
|
|
* @returns {Download}
|
|
* @typedef {String} Link
|
|
* @typedef {Object} Download
|
|
* @property {String} status
|
|
* @property {Path} file
|
|
*/
|
|
async function downloadImage(url, destPath, maxRetries = 3)
|
|
{
|
|
let retries = 0;
|
|
if(!shouldKeepOnDownloading) throw new Error('UserStoped');
|
|
while (retries < maxRetries && shouldKeepOnDownloading) {
|
|
try
|
|
{
|
|
const response = await fetch(url);
|
|
const dir = path.dirname(destPath);
|
|
if (!response.ok) throw new Error(`Failed to download image. Status Code: ${response.status}`);
|
|
if (!fs.existsSync(dir)) {
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
const fileStream = fs.createWriteStream(destPath);
|
|
await new Promise((resolve, reject) =>
|
|
{
|
|
const stream = Stream.Readable.fromWeb(response.body).pipe(fileStream);
|
|
stream.on('error', reject);
|
|
fileStream.on('finish', () => {
|
|
fileStream.close();
|
|
resolve();
|
|
});
|
|
});
|
|
return {status:'Downloaded', file:destPath}
|
|
} catch (error) {
|
|
console.error(`Error downloading image: ${destPath}, Retry ${retries + 1}/${maxRetries}`);//, error);
|
|
retries++;
|
|
if(retries == maxRetries)
|
|
return {status:'Failed', file:destPath}
|
|
}
|
|
}
|
|
}
|