# robots.txt

User-agent: *
Allow: /

# Protect WordPress core directories from indexing
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Prevent indexing of unnecessary files
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /readme.html
Disallow: /license.txt

# Prevent indexing of internal search pages
Disallow: /?s=
Disallow: /search/

# Optional: Block common temporary or backup files
Disallow: /*.sql$
Disallow: /*.zip$
Disallow: /*.tar$
Disallow: /*.gz$
Disallow: /*.bak$
Disallow: /*.old$
Disallow: /*.log$