ROBOTS.TXT Generator

Standalone, zero-dependency JavaScript implementation powering this tool. Free to inspect, copy, and build upon.

let groupCounter = 0;
  let groups = []; // [{id, agent, paths:[{type,path}], delay}]

  function addRuleGroup(config) {
    const id = ++groupCounter;
    const group = config || { id, agent: '*', paths: [{ type: 'Disallow', path: '' }], delay: '' };
    group.id = id;
    groups.push(group);
    renderGroups();
    updatePreview();
    return id;
  }

  function removeGroup(id) {
    groups = groups.filter(g => g.id !== id);
    renderGroups();
    updatePreview();
  }

  function addPath(groupId) {
    const g = groups.find(g => g.id === groupId);
    if (g) g.paths.push({ type: 'Disallow', path: '' });
    renderGroups();
    updatePreview();
  }

  function removePath(groupId, pathIdx) {
    const g = groups.find(g => g.id === groupId);
    if (g && g.paths.length > 1) g.paths.splice(pathIdx, 1);
    renderGroups();
    updatePreview();
  }

  function renderGroups() {
    const container = document.getElementById('ruleGroups');
    container.innerHTML = '';
    groups.forEach((g, gi) => {
      const div = document.createElement('div');
      div.className = 'rule-group';
      div.innerHTML = `
        <div class="rule-group-header">
          <span class="rule-group-title">Rule Group ${gi + 1}</span>
          <button class="remove-group-btn" onclick="removeGroup(${g.id})">Remove</button>
        </div>
        <div class="field">
          <label>User-Agent</label>
          <input type="text" value="${escVal(g.agent)}" placeholder="e.g. * or Googlebot"
            oninput="updateGroupField(${g.id}, 'agent', this.value)" />
        </div>
        <div class="field">
          <label>Rules (Allow / Disallow)</label>
          <ul class="paths-list" id="paths-${g.id}">
            ${g.paths.map((p, pi) => `
              <li class="path-item">
                <select class="path-type-select" onchange="updatePath(${g.id}, ${pi}, 'type', this.value)">
                  <option value="Allow" ${p.type === 'Allow' ? 'selected' : ''}>Allow</option>
                  <option value="Disallow" ${p.type === 'Disallow' ? 'selected' : ''}>Disallow</option>
                </select>
                <input type="text" value="${escVal(p.path)}" placeholder="/path/to/page"
                  oninput="updatePath(${g.id}, ${pi}, 'path', this.value)" />
                <button class="remove-path-btn" onclick="removePath(${g.id}, ${pi})" title="Remove">&#xd7;</button>
              </li>
            `).join('')}
          </ul>
          <button class="add-path-btn" onclick="addPath(${g.id})">+ Add Rule</button>
        </div>
        <div class="field" style="margin-bottom:0;">
          <label>Crawl-Delay (seconds, optional)</label>
          <input type="number" value="${escVal(g.delay)}" placeholder="e.g. 10" min="0" max="120"
            oninput="updateGroupField(${g.id}, 'delay', this.value)" style="width:140px;" />
        </div>
      `;
      container.appendChild(div);
    });
  }

  function updateGroupField(id, field, val) {
    const g = groups.find(g => g.id === id);
    if (g) g[field] = val;
    updatePreview();
  }

  function updatePath(groupId, pathIdx, field, val) {
    const g = groups.find(g => g.id === groupId);
    if (g && g.paths[pathIdx]) g.paths[pathIdx][field] = val;
    updatePreview();
  }

  function buildRobotsText() {
    const lines = [];
    lines.push('# Generated by toolpad.cc/robotsgen/');

    const sitemap = document.getElementById('sitemapUrl').value.trim();
    const globalDelay = document.getElementById('crawlDelay').value.trim();

    groups.forEach(g => {
      lines.push('');
      lines.push('User-agent: ' + (g.agent.trim() || '*'));
      g.paths.forEach(p => {
        if (p.path !== undefined) {
          lines.push(p.type + ': ' + p.path);
        }
      });
      const delay = (g.delay || '').trim() || globalDelay;
      if (delay) lines.push('Crawl-delay: ' + delay);
    });

    if (sitemap) {
      lines.push('');
      lines.push('Sitemap: ' + sitemap);
    }

    return lines.join('\n').trim();
  }

  function updatePreview() {
    const text = buildRobotsText();
    // Syntax highlight
    const highlighted = text
      .split('\n')
      .map(line => {
        if (line.startsWith('#')) return '<span class="cmt">' + esc(line) + '</span>';
        const m = line.match(/^(User-agent|Allow|Disallow|Crawl-delay|Sitemap|Host):\s*(.*)$/);
        if (m) return '<span class="key">' + esc(m[1]) + ':</span> <span class="val">' + esc(m[2]) + '</span>';
        return esc(line);
      })
      .join('\n');
    document.getElementById('robotsOutput').innerHTML = highlighted;
  }

  function esc(s) {
    return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
  }
  function escVal(s) {
    return (s || '').replace(/"/g,'&quot;').replace(/'/g,'&#39;');
  }

  function copyOutput() {
    const text = buildRobotsText();
    navigator.clipboard.writeText(text).then(() => {
      const btn = document.querySelector('.output-actions .btn-primary');
      const orig = btn.textContent;
      btn.textContent = 'Copied!';
      setTimeout(() => { btn.textContent = orig; }, 2000);
    });
  }

  function downloadOutput() {
    const text = buildRobotsText();
    const blob = new Blob([text], { type: 'text/plain' });
    const a = document.createElement('a');
    a.href = URL.createObjectURL(blob);
    a.download = 'robots.txt';
    a.click();
    URL.revokeObjectURL(a.href);
  }

  function resetAll() {
    groups = [];
    groupCounter = 0;
    document.getElementById('sitemapUrl').value = '';
    document.getElementById('crawlDelay').value = '';
    renderGroups();
    addRuleGroup();
  }

  const PRESETS = {
    'allow-all': {
      sitemap: '',
      delay: '',
      groups: [{ agent: '*', paths: [{ type: 'Allow', path: '/' }], delay: '' }]
    },
    'block-all': {
      sitemap: '',
      delay: '',
      groups: [{ agent: '*', paths: [{ type: 'Disallow', path: '/' }], delay: '' }]
    },
    'block-bots': {
      sitemap: '',
      delay: '',
      groups: [
        { agent: '*', paths: [{ type: 'Allow', path: '/' }], delay: '' },
        { agent: 'GPTBot', paths: [{ type: 'Disallow', path: '/' }], delay: '' },
        { agent: 'ChatGPT-User', paths: [{ type: 'Disallow', path: '/' }], delay: '' },
        { agent: 'CCBot', paths: [{ type: 'Disallow', path: '/' }], delay: '' },
        { agent: 'anthropic-ai', paths: [{ type: 'Disallow', path: '/' }], delay: '' },
        { agent: 'Claude-Web', paths: [{ type: 'Disallow', path: '/' }], delay: '' },
        { agent: 'PerplexityBot', paths: [{ type: 'Disallow', path: '/' }], delay: '' },
        { agent: 'Bytespider', paths: [{ type: 'Disallow', path: '/' }], delay: '' },
      ]
    },
    'wordpress': {
      sitemap: 'https://example.com/sitemap.xml',
      delay: '',
      groups: [
        {
          agent: '*',
          paths: [
            { type: 'Disallow', path: '/wp-admin/' },
            { type: 'Allow', path: '/wp-admin/admin-ajax.php' },
            { type: 'Disallow', path: '/wp-includes/' },
            { type: 'Disallow', path: '/wp-content/plugins/' },
            { type: 'Disallow', path: '/?s=' },
            { type: 'Disallow', path: '/search/' },
            { type: 'Allow', path: '/wp-content/uploads/' },
          ],
          delay: ''
        }
      ]
    }
  };

  function applyPreset(name) {
    const p = PRESETS[name];
    if (!p) return;
    groups = [];
    groupCounter = 0;
    document.getElementById('sitemapUrl').value = p.sitemap || '';
    document.getElementById('crawlDelay').value = p.delay || '';
    p.groups.forEach(g => {
      const id = ++groupCounter;
      groups.push({ ...g, id, paths: g.paths.map(pp => ({ ...pp })) });
    });
    renderGroups();
    updatePreview();
  }

  // Init with default group
  addRuleGroup({ agent: '*', paths: [{ type: 'Disallow', path: '' }], delay: '' });