#-----------------------------------------------------------------------------
# robots.txt
#-----------------------------------------------------------------------------
# This file is under RCS control; use 'co -l robots.txt' (unix command) to
# reserve it for modification, and 'ci -u robots.txt' to replace it when you
# are finished.  Use 'rcsdiff robots.txt' to see differences between the
# working version and the checked-in version, and 'rlog robots.txt' to get a
# summary of changes that have been made.
#
# $Id: robots.txt,v 1.7 2009/05/06 06:11:09 kos025 Exp $
#-----------------------------------------------------------------------------

User-Agent: ATNFCrawler

Allow: /_Mail_Archives/                   # Mail_Archives
Disallow: /_Server/                          # wwwstat server statistics files
Allow: /aips++/                           # huge AIPS++ document tree
Disallow: /cgi-bin/                          # for internal use
Disallow: /common/                           # all common files
Allow: /computing/news/nexus
Allow: /computing/nexus
Allow: /computing/policy.html             # big documentation tree
Allow: /computing/software/Midas/         # big documentation tree
Allow: /computing/software/aips++/        # big documentation tree
Allow: /computing/software/miriad/        # big documentation tree
Disallow: /database/                         # database files, no html
Disallow: /feedback/
Disallow: /images/library/
Disallow: /ims/
Disallow: /includes/
#Disallow: /internal/                          # should we allow any scans here?
Disallow: /internal/computing/web/statistics/  # Contains links to pages we may not want searched
Disallow: /internal/computing/web/test/
Disallow: /internal/computing/pmwiki/
Disallow: /internal/test/
Disallow: /java/                             # test progs only
Disallow: /linkscan/
Allow: /local/                            # restricted local pages
Allow: /management/
Allow: /meetings/
Allow: /mnrf/restricted/
Allow: /news/webnew/                      # What's New pages **WHY DISALLOW?
Allow: /overview/computing/
Allow: /overview/management/
Disallow: /people/Neil.Killeen/test/
Disallow: /people/Neil.Killeen/test/ELARA_3/mitk/
Disallow: /people/allrite/
Disallow: /people/dbarnes/.private/
Allow: /people/mcalabre/data/
Allow: /people/rnorris/COLA/data/
Allow: /projects/mnrf/restricted/
Allow: /research/pulsar/psrlunch/


User-Agent: *

Disallow: /_Mail_Archives/                   # Mail_Archives
Disallow: /_Server/                          # wwwstat server statistics files
Disallow: /aips++/                           # huge AIPS++ document tree
Disallow: /cgi-bin/                          # for internal use
Disallow: /common/                           # all common files
Disallow: /computing/news/nexus
Disallow: /computing/nexus
Disallow: /computing/policy.html             # big documentation tree
Disallow: /computing/software/Midas/         # big documentation tree
Disallow: /computing/software/aips++/        # big documentation tree
Disallow: /computing/software/miriad/        # big documentation tree
Disallow: /database/                         # database files, no html
Disallow: /feedback/
Disallow: /images/library/
Disallow: /ims/
Disallow: /includes/
#Disallow: /internal/                          # should we allow any scans here?
Disallow: /internal/computing/web/statistics/  # Contains links to pages we may not want searched
Disallow: /internal/computing/web/test/
Disallow: /internal/computing/pmwiki/
Disallow: /internal/test/
Disallow: /java/                             # test progs only
Disallow: /linkscan/
Disallow: /local/                            # restricted local pages
Disallow: /management/
Disallow: /meetings/
Disallow: /mnrf/restricted/
Disallow: /news/webnew/                      # What's New pages **WHY DISALLOW?
Disallow: /overview/computing/
Disallow: /overview/management/
Disallow: /people/Neil.Killeen/test/
Disallow: /people/Neil.Killeen/test/ELARA_3/mitk/
Disallow: /people/allrite/
Disallow: /people/dbarnes/.private/
Disallow: /people/mcalabre/data/
Disallow: /people/rnorris/COLA/data/
Disallow: /projects/mnrf/restricted/
Disallow: /research/pulsar/psrlunch/