User-agent: *
  Disallow: /register/
  Disallow: /register
  Disallow: /wp-json/theme/v1/ad/
  Disallow: /wp-json/oembed/
  Disallow: /xmlrpc.php
  Disallow: /media-monitoring/

  # Allow important SEO bots to crawl most content
  User-agent: Googlebot
  Allow: /

  User-agent: Bingbot
  Allow: /

  # Block aggressive crawlers entirely from expensive endpoints
  User-agent: AmazonBot
  Disallow: /register/
  Disallow: /wp-json/

  User-agent: SemrushBot
  Disallow: /register/
  Disallow: /wp-json/

  User-agent: AhrefsBot
  Disallow: /register/
  Disallow: /wp-json/