# ByeGPT
User-Agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: sentibot
Disallow: /

# Adios Google Bard/Gemini AI as well
# Does not impact normal Google crawlers
# See: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
User-agent: Google-Extended
Disallow: /

# More AI that needs to go eff itself
# I don't expect them to obey this but worth a shot
# See: https://udger.com/resources/ua-list/bot-detail?bot=ClaudeBot
User-Agent: ClaudeBot
Disallow: /

User-Agent: Claude-Web
Disallow: /

User-Agent: PiplBot
Disallow: /

User-Agent: anthropic-ai
Disallow: /

User-Agent: *
# Block pages with no unique content
Disallow: /startTopic/
Disallow: /discover/unread/
Disallow: /markallread/
Disallow: /online/
Disallow: /discover/
Disallow: /leaderboard/
Disallow: /search/
Disallow: /*?advancedSearchForm=
Disallow: /register/
Disallow: /lostpassword/
Disallow: /login/

# Block faceted pages and 301 redirect pages
Disallow: /*?sortby=
Disallow: /*?filter=
Disallow: /*?tab=
Disallow: /*?do=
Disallow: /*ref=
Disallow: /*?forumId*

# Block profile pages as these have little unique value, consume a lot of crawl time and contain hundreds of 301 links
# Invision have removed this recently (01/2024) but we don't allow guests to see profiles, so might as well save the bots who
# respect this file some effort
Disallow: /profile/

# Bots who respect this please be nicer
# Crawl every second and not any faster
Crawl-delay: 1

# Sitemap URL
Sitemap: https://gtaforums.com/sitemap.php