# robots.txt 
User-agent: *
Disallow: 
Disallow: /files/
Disallow: /images/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /oldfiles/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /Scripts/
Disallow: /sites/
Disallow: /themes/
Disallow: /tmp/

User-Agent: MJ12bot
Disallow: