# Programmatic bots (these will ignore robots.txt but included anyway) User-agent: python-requests Disallow: / User-agent: python-httpx Disallow: / User-agent: aiohttp Disallow: / User-agent: urllib Disallow: / User-agent: urllib2 Disallow: / User-agent: curl Disallow: / User-agent: Wget Disallow: / User-agent: okhttp Disallow: / User-agent: java Disallow: / User-agent: httpclient Disallow: / User-agent: libwww-perl Disallow: / User-agent: PHP Disallow: / User-agent: Ruby Disallow: / User-agent: Perl Disallow: / User-agent: Go-http-client Disallow: / User-agent: PowerShell Disallow: / User-agent: node-fetch Disallow: / User-agent: axios Disallow: /