# Crawl settings Crawl-delay: 10 # --------------------------- # Standard Rules # --------------------------- User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Allow llms.txt Allow: /llms.txt # Yoast Sitemap Sitemap: https://ptwlaw.com.au/sitemap_index.xml # --------------------------- # LLM / AI Crawler Instructions # --------------------------- User-agent: ChatGPT-User Allow: / User-agent: GPTBot Allow: / User-agent: ClaudeBot Allow: / User-agent: PerplexityBot Allow: /