User-agent: * Disallow: /hukildas/ # Allow the Grapeshot crawler full access User-agent: grapeshot Disallow: