cli.js

#!/usr/bin/env node
import fs from 'fs/promises';
import path from 'path';

const COMMUNITY_LIST_URL = 'https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt';

const colors = {
    reset: '\x1b[0m',
    green: '\x1b[32m',
    cyan: '\x1b[36m',
    red: '\x1b[31m',
    yellow: '\x1b[33m',
    magenta: '\x1b[35m'
};

const logo = `
${colors.cyan}  _____       _           _               
 |  __ \\     | |         | |              
 | |__) |___ | |__   ___ | |_  ___ __  __ 
 |  _  // _ \\| '_ \\ / _ \\| __|/ _ \\\\ \\/ / 
 | | \\ \\ (_) | |_) | (_) | |_|  __/ >  <  
 |_|  \\_\\___/|_.__/ \\___/ \\__|\\___|/_/\\_\\ ${colors.reset}
`;

async function fileExists(filePath) {
    try {
        await fs.access(filePath);
        return true;
    } catch {
        return false;
    }
}

async function initConfig() {
    console.log(logo);
    const template = {
        sitemap: "https://yourdomain.com/sitemap.xml",
        allowBots: ["Googlebot", "Bingbot"],
        blockBots: ["BadBot", "AnnoyingScraper"],
        output: ""
    };
    
    try {
        await fs.writeFile('robotex.json', JSON.stringify(template, null, 2));
        console.log(`${colors.green}[X] Successfully created robotex.json!${colors.reset}`);
        console.log(`${colors.cyan}[i] Edit this file to add your custom rules, then run 'robotex' again.${colors.reset}\n`);
    } catch (err) {
        console.error(`${colors.red}[!] Error creating robotex.json: ${err.message}${colors.reset}`);
    }
}

async function generateRobotsTxt() {
    const args = process.argv.slice(2);
    
    if (args.includes('init')) {
        return initConfig();
    }

    const isDryRun = args.includes('--dry-run');

    console.log(logo);
    if (isDryRun) {
        console.log(`${colors.yellow}>> DRY RUN MODE: No files will be saved <<${colors.reset}\n`);
    }

    const spinnerFrames = ['-', '\\', '|', '/'];
    let i = 0;
    const spinner = setInterval(() => {
        process.stdout.write(`\r${colors.yellow}[${spinnerFrames[i++ % spinnerFrames.length]}] Fetching the latest community blocklist...${colors.reset}`);
    }, 100);

    try {
        const response = await fetch(COMMUNITY_LIST_URL);
        if (!response.ok) throw new Error('Failed to fetch community list');
        const communityData = await response.text();

        clearInterval(spinner);
        process.stdout.write('\r\x1b[K');
        console.log(`${colors.green}[X] Downloaded latest community blocklist!${colors.reset}`);

        let config = {};
        try {
            const configFile = await fs.readFile('robotex.json', 'utf-8');
            config = JSON.parse(configFile);
            console.log(`${colors.green}[X] Found local robotex.json config!${colors.reset}`);
        } catch (err) {
            console.log(`${colors.cyan}[i] No local robotex.json found, skipping custom rules.${colors.reset}`);
        }

        let finalOutput = "# Generated by Robotex\n\n";

        if (config.sitemap) {
            finalOutput += `Sitemap: ${config.sitemap}\n\n`;
        } else {
            const dirsToCheck = ['public/sitemap.xml', 'static/sitemap.xml', 'sitemap.xml'];
            for (const spath of dirsToCheck) {
                if (await fileExists(spath)) {
                    console.log(`${colors.cyan}[i] Auto-discovered ${spath}! Added placeholder to robots.txt${colors.reset}`);
                    finalOutput += `# Auto-discovered local sitemap\nSitemap: https://YOUR-DOMAIN.com/${spath.replace('public/', '').replace('static/', '')}\n\n`;
                    break;
                }
            }
        }

        if (config.allowBots && config.allowBots.length > 0) {
            finalOutput += "# Custom Allowed Bots\n";
            config.allowBots.forEach(bot => {
                finalOutput += `User-agent: ${bot}\nAllow: /\n\n`;
            });
        }

        if (config.blockBots && config.blockBots.length > 0) {
            finalOutput += "# Custom Blocked Bots\n";
            config.blockBots.forEach(bot => {
                finalOutput += `User-agent: ${bot}\nDisallow: /\n\n`;
            });
        }

        finalOutput += "# --- Community AI Bot Blocklist ---\n";
        finalOutput += communityData;

        let outputPath = 'robots.txt';
        if (config.output) {
            outputPath = config.output;
        } else {
            const commonDirs = ['public', 'static'];
            for (const dir of commonDirs) {
                try {
                    const stats = await fs.stat(dir);
                    if (stats.isDirectory()) {
                        outputPath = path.join(dir, 'robots.txt');
                        break;
                    }
                } catch (err) {}
            }
        }

        if (isDryRun) {
            console.log(`\n${colors.magenta}=== PREVIEW OF ${outputPath} ===${colors.reset}\n`);
            console.log(finalOutput);
            console.log(`${colors.magenta}====================================${colors.reset}\n`);
        } else {
            await fs.writeFile(outputPath, finalOutput);
            console.log(`\n${colors.green}Success! robots.txt generated at: ${outputPath}${colors.reset}\n`);
        }

    } catch (error) {
        clearInterval(spinner);
        process.stdout.write('\r\x1b[K');
        console.error(`${colors.red}[!] Error generating robots.txt: ${error.message}${colors.reset}`);
    }
}

generateRobotsTxt();