#-----------------------------------------------------------------------------
# robots.txt
#-----------------------------------------------------------------------------
# This file is under RCS control; use 'co -l robots.txt' (unix command) to
# reserve it for modification, and 'ci -u robots.txt' to replace it when you
# are finished.  Use 'rcsdiff robots.txt' to see differences between the
# working version and the checked-in version, and 'rlog robots.txt' to get a
# summary of changes that have been made.
#
# $Id: robots.txt,v 1.11 2010/06/28 00:28:47 root Exp $
#-----------------------------------------------------------------------------

User-Agent: *

Disallow: /_Server/                          # wwwstat server statistics files
Disallow: /aips++/                           # huge AIPS++ document tree
Disallow: /cgi-bin/                          # for internal use
Disallow: /common/                           # all common files
Disallow: /computing/news/nexus
Disallow: /computing/nexus
Disallow: /computing/software/aips++/        # big documentation tree
Disallow: /images/
Disallow: /ims/
Disallow: /includes/
Disallow: /internal/computing/web/statistics/  # Contains links to pages we may not want searched
Disallow: /internal/computing/web/test/
Disallow: /internal/computing/pmwiki/
Disallow: /internal/test/
Disallow: /news/webnew/                      # What's New pages **WHY DISALLOW?
Disallow: /people/mcalabre/data/
Disallow: /research/pulsar/psrlunch/
Disallow: /research/pulsar/psrcat/proc_form.php   # catalog access interface