User-agent: *   # directed to all spiders
Disallow: /

User-agent: gsa-crawler
Allow: /intranet/