User-agent: *
Crawl-delay: 1
Disallow: /admin
Disallow: /advertisements
Disallow: /artists
Disallow: /artist_commentaries
Disallow: /artist_commentary_versions
Disallow: /artist_versions
Disallow: /bans
Disallow: /comment_votes
Disallow: /comment/
Disallow: /comments
Disallow: /comments.atom
Disallow: /comments/*.atom
Disallow: /counts
Disallow: /data/
Disallow: /delayed_jobs
Disallow: /deleted_posts
Disallow: /dmails
Disallow: /explore
Disallow: /favorite
Disallow: /forum_topics.atom
Disallow: /forum_posts
Disallow: /forum_posts.atom
Disallow: /forum_topics/*.atom
Disallow: /forum_posts/*.atom
Disallow: /iqdb_queries
Disallow: /ip_bans
Disallow: /janitor_trials
Disallow: /maintenance
Disallow: /m/
Disallow: /mod_actions
Disallow: /moderator
Disallow: /news_updates
Disallow: /note_versions
Disallow: /notes
Disallow: /pool_elements
Disallow: /pool_orders
Disallow: /pool_versions
Disallow: /popular
Disallow: /post_appeals
Disallow: /post_flags
Disallow: /post_sets
Disallow: /post_versions
Disallow: /post_votes
Disallow: /post_events
Disallow: /post_replacements
Disallow: /post/
Disallow: /posts.atom
Disallow: /posts/random
Disallow: /posts/copy_notes
Disallow: /posts/revert
Disallow: /posts/show_seq
Disallow: /posts*?*tags=*
Disallow: /posts*?*q=*
Disallow: /posts*?*md5*
Disallow: /posts*?*limit*
Disallow: /posts*?*random*
Disallow: /posts*?*pool_id*
Disallow: /posts*?*post_set_id*
Disallow: /posts.json
Disallow: /posts.xml
Disallow: /posts/*/favorites
Disallow: /reports
Disallow: /session
Disallow: /sources
Disallow: /static
Disallow: /tag_alias_corrections
Disallow: /tag_alias_requests
Disallow: /tag_aliases
Disallow: /tag_corrections
Disallow: /tag_implication_requests
Disallow: /tag_implications
Disallow: /tags
Disallow: /tickets
Disallow: /takedowns
Disallow: /uploads
Disallow: /user_feedback
Disallow: /user/
Disallow: /users
Disallow: /wiki/
Disallow: /wiki_page_versions
Disallow: /wiki_pages*?*title=*
Disallow: /wiki_pages/new
Disallow: /wiki_pages/revert
Disallow: /wiki_pages/show_or_new

User-agent: MJ12bot
User-agent: dotbot
User-agent: coccocbot-web
User-agent: BLEXBot
User-agent: HTTrack
User-agent: Slurp
User-agent: Baiduspider
User-agent: Yandex
User-agent: magpie-crawler
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: AhrefsBot
Disallow: /