# example spider filder by domain regular expressions courtesy of OSU Libraries
# https://raw.github.com/osulibraries/DSpace/osukb/dspace/config/Spiders-DomainName.txt
(.*)\.fastsearch\.net\.
(.*)\.scoutjet\.com\.
(.*)\.yahoo\.com\.
crawl(.*)\.exabot\.com\.
crawl-(.*)-googlebot\.com\.
crawler(.*)\.ask\.com\.
discobot-(.*)\.discoveryengine\.com\.
localhost\.
spider(.*)\.yandex\.ru\.
spider-(.*)\.yandex\.com\.