mangaReader/test2.js

const mongoose = require('mongoose');
const mongoURI = 'mongodb://mongoadmin:something@192.168.71.137:27017';

mongoose.connect(mongoURI);
mongoose.connection.on('connected', ()=>{console.log('MongoDB - Connected')})
        .on('disconnected', ()=>{console.log('MongoDB - Disconnect')})
        .on('error', (error)=>console.log('Mongoose Error:', error));

const path = require('path');
const fs = require('fs');
const { Stream } = require('stream');
const {Manga, Chapter} = require('./lib');
const FavModel = require('./models/favorite')
const EventEmitter = require('events');
const Emitter = new EventEmitter();
const BaseDir  = './public/Saved'

var Queue = [];
var slots = 2;
updateQueue();
var queueUpdateTimer = setInterval(() => {
    updateQueue();
}, 5*60*1000);


process.on('message', m=>
{
    if(m.action==='add')
    {
        clearInterval(queueUpdateTimer);
        updateQueue();
        queueUpdateTimer = setInterval(() => {
            updateQueue();
        }, 5*60*1000);
        return 
    }
    
    if(m.action==='remove') return removeFromQueue(m.id) 
})
function removeFromQueue(id)
{
    Queue = Queue.filter(manga => manga._id !== id);
}
Emitter.on('queueUpdate', ()=>
{
    if(!Queue[0]) return 
    if(!Queue[0].inProgress)
    {
        Queue[0].inProgress = true;
        download(Queue[0])
    }
} );

async function updateQueue()
{
    const Scanlators = fs.readdirSync(BaseDir);
    const Favs = await FavModel.find();
    let Mangas = [];
    for(let i =0; i<Scanlators.length; i++)
    {
        let mangas = fs.readdirSync(path.join(BaseDir, Scanlators[i]));
        if(mangas.length < 0) continue 
        for(var j = 0; j<mangas.length; j++)
        {
            let chapters = fs.readdirSync(path.join(BaseDir, Scanlators[i], mangas[j]));
            if(chapters.length==0) continue
            chapters.sort((a, b) => { return parseFloat(a.split('_')[1]) -  parseFloat(b.split('_')[1]) });
            let latestDownloaded = parseFloat(chapters[chapters.length-1].split('_')[1]);
            let manga = {scanlator:Scanlators[i], title:mangas[j], latestDownloaded}
            Mangas.push(manga)
        }
    }
    let FavedAndDownloaded = [];
    let FavedButNotDownloaded = null;
    for (let i = 0; i < Mangas.length; i++) 
    {
        FavedAndDownloaded = FavedAndDownloaded.concat(Favs.filter(item => 
            {
                return (item.scanlator === Mangas[i].scanlator && item.title === Mangas[i].title);
            }).map((matchedItem) =>{
                return ({
                    ...matchedItem._doc, 
                    latestDownloaded: Mangas[i].latestDownloaded
                })
            })
        );
    }
    FavedButNotDownloaded = Favs.filter(fav => 
        {
            // return !FavedAndDownloaded.includes(fav)
            return !FavedAndDownloaded.some(fad => fad._id === fav._id);
        });
    if(FavedButNotDownloaded) 
    {
        for(let i = 0; i < FavedButNotDownloaded.length; i++)
        {
            (async(i)=>
            {
                const {title, link, scanlator, _id} = FavedButNotDownloaded[i];
                const manga = await new Manga(scanlator, link, title).get();
                let id = _id.toString();
                addToQueue(manga, id);
            })(i)
        }
    }
    if(FavedAndDownloaded)
    {
        for(let i = 0; i<FavedAndDownloaded.length; i++)
        {
            (async(i)=>
            {
                const {title, link, scanlator, latestDownloaded, _id} = FavedAndDownloaded[i];
                const  manga = await new Manga(scanlator, link, title).get();
                let id = _id.toString();
                checkForMissingFiles(manga)
                if(latestDownloaded<manga.latestChap) addToQueue(manga, id, latestDownloaded);
            })(i);
        }
    }
    function addToQueue(manga, _id,startFrom=0)
    {
        const List = manga.List.filter(item => item.num > startFrom);
        Queue.push({...manga, _id, List, startFrom})
        Emitter.emit('queueUpdate', '');
    }
}
async function checkForMissingFiles(manga)
{
    const {scanlator, title, List} = manga;
    let auxList = List.sort((a,b)=>{return a.num - b.num})
    for(var i = 0; i<auxList.length; i++)
    {
        const {link} = auxList[i];
        const destPath = path.join(BaseDir, scanlator, title, 'CH_'+(i+1));
        const chapterDownloaded = fs.existsSync(destPath);
        if(!chapterDownloaded) continue;
        const chapterDir = fs.readdirSync(destPath);
        const chapter = await new Chapter(scanlator, link, title, i+1).get();
        let auxChapterDir = chapterDir.map(item=>{return item.split('.')[0]})
        var auxChapterList = chapter.List.map(item=>{return item.split('/')[item.split('/').length-1].split('.')[0]})
        const missingImages = auxChapterList.filter(item=>!auxChapterDir.includes(item));
        if(missingImages.length == 0) continue;
        for(var j = 0; j<missingImages.length; j++)
        {
            const imageLink = chapter.List.filter(item=>item.split('/')[item.split('/').length-1].split('.')[0] == missingImages[j])[0]
            const destination = `${destPath}/${imageLink.split('/')[(imageLink.split('/').length - 1)]}`
            const download = await downloadImage(imageLink,destination, 5);
            console.log('MissingImage:',scanlator, title, 'chapter',i+1, download.status)
        }        
    }
}
async function download(manga) 
{
    if(slots <=0) return 
    slots--;
    const { scanlator, title} = manga;
    const List = manga.List.sort((a, b) => { return a.num -  b.num });
    for (let i = 0; i < List.length; i++) 
    {
        const ch = List[i];
        if (isNaN(ch.num)) continue;
        const { num, link } = ch;
        const chapter = await new Chapter(scanlator, link, title, num).get();
        // console.log(chapter)
        if (!chapter?.List?.length) continue;
        await downloadChapter(chapter);
    }
    Queue.shift();
    slots++;
    Emitter.emit('queueUpdate', '');
}
async function downloadChapter(chapter)
{
    const {List, scanlator, title, chNum } = chapter;
    const images = List; 
    for (let i = 0; i < images.length; i++) {
        const img = images[i];
        const destination = `./public/Saved/${scanlator}/${title}/CH_${chNum}/${img.split('/')[(img.split('/').length - 1)]}`;
        try {
            const downloaded = await downloadImage(img, destination);
            console.log(`Downloaded: Scanlator:${scanlator}; Manga: ${title} Chapter:${chNum}; IMG:${img.split('/')[(img.split('/').length - 1)]} ${downloaded.status}`); 
        } catch (error) {
            console.error(`Error downloading ${img}: ${error}`);
        }
    }
    return {chNum}
}
/**
 * 
 * @param {Link} url 
 * @param {Path} destPath 
 * @param {Number} maxRetries 
 * @returns {Download} 
 * @typedef {String} Link
 * @typedef {Object} Download
 * @property {String} status
 * @property {Path} file
 */
async function downloadImage(url, destPath, maxRetries = 3)
{
    let retries = 0;
    while (retries < maxRetries) {
        try
        {
            const response = await fetch(url);
            const dir = path.dirname(destPath);
            if (!response.ok) throw new Error(`Failed to download image. Status Code: ${response.status}`);
            if (!fs.existsSync(dir)) {
                fs.mkdirSync(dir, { recursive: true });
            }
            const fileStream = fs.createWriteStream(destPath);
            await new Promise((resolve, reject) => 
            {
                const stream = Stream.Readable.fromWeb(response.body).pipe(fileStream);
                stream.on('error', reject);
                fileStream.on('finish', () => {
                    fileStream.close();
                    resolve();
                });
            });
            return {status:'Downloaded', file:destPath}
        } catch (error) {
            console.error(`Error downloading image: ${destPath}, Retry ${retries + 1}/${maxRetries}`);//, error);
            retries++;     
            if(retries == maxRetries)
            return {status:'Failed', file:destPath}
        }
    }
}