web/robots.txt

   1 # Robots.txt: Configure which spiders can crawl this site
   2
   3 # Why is this server crawling my site?
   4 User-agent: panscient_data_services.demarc.cogentco.com
   5 Disallow: /
   6 User-agent: Inktomi
   7 Disallow: /
   8 User-agent: BaiDuSpider
   9 Disallow: /
  10 User-agent: crawl
  11 Disallow: /
  12 User-agent: GigaBot
  13 Disallow: /
  14 User-agent: arks
  15 Disallow: /
  16 User-agent: EchO!
  17 Disallow: /
  18 User-agent: Viola
  19 Disallow: /
  20 User-agent: hit
  21 Disallow: /
  22 User-agent: WISENutbot
  23 Disallow: /
  24 User-agent: BBot
  25 Disallow: /
  26 User-agent: spider
  27 Disallow: /
  28 User-agent: psbot
  29 Disallow: /
  30 User-agent: SurveyBot
  31 Disallow: /
  32
  33 # Allow all others not listed above
  34 User-agent: *
  35 Disallow: /Backgrounds
  36 Disallow: /bin
  37 Disallow: /binme
  38 Disallow: /doc
  39 Disallow: /Fonts
  40 Disallow: /gallery
  41 Disallow: /Icons
  42 Disallow: /Images
  43 Disallow: /INS
  44 Disallow: /Legal
  45 Disallow: /msoffice
  46 Disallow: /Music
  47 Disallow: /Olga
  48 Disallow: /Personal
  49 Disallow: /Pictures
  50 Disallow: /Senators
  51 Disallow: /Software
  52 Disallow: /Sounds
  53 Disallow: /Warsaw
  54 Disallow: /Wedding
  55 Disallow: /jinzora
  56 Disallow: /jinzora2
  57 Disallow: /blogs/Status
  58 Disallow: /IBM
  59 Disallow: /Broadcom