# General guidelines for all search engines
User-agent: *

# Block search and query parameters
Disallow: /?s=*
Disallow: /search/
Disallow: /search-result/

# Block sensitive or internal paths
Disallow: /admin/
Disallow: /user/
Disallow: /comment/active/email/
Disallow: /FormRule/
Disallow: /rate-review/step3/
Disallow: /awardPayment/paymentStatus/
Disallow: /properties/emaildemo/
Disallow: /properties/viewdemo/
Disallow: /404/
Disallow: /noDataFound/
Disallow: /comming-soon/
Disallow: /printForm/
Disallow: /actions/
Disallow: /inc/
Disallow: /api/
Disallow: /pages/
Disallow: /pages/download-qr
Disallow: /rate-review/
Disallow: /remind-branch-email/
Disallow: /update-branch-staff-new/
Disallow: /upload-your-photo/
Disallow: /update-branch/
Disallow: /test-agent-123456/
Disallow: /view-branch/test-agent-123456/alternative-branch/
Disallow: /view-branch/test-agent-123456/original/

Disallow: /rss/
Disallow: /properties/rss/
Disallow: /*/from_sms/
Disallow: /application/cronjobs/
Disallow: /cronjobs/
Disallow: /people-directory/
Disallow: /properties/ajax/
Disallow: /application/
Disallow: /feature-review/*
Disallow: /feature-review/

# Block specific paths
Disallow: /agent/
Disallow: /properties/report/print/branch/40164/
 
# Allow specific files
Allow: /inc/floating-valuation-all.phtml

# Allow public-facing routes
Allow: /
Allow: /*
Allow: /find-agent/
Allow: /view-branch/
Allow: /rate-review/
Allow: /properties/
Allow: /league-tables/
Allow: /agent/
Allow: /suppliers/
Allow: /house-prices/
Allow: /free-instant-valuation/
Allow: /review/
Allow: /calculator/

Allow: /review-print/
Allow: /supplier/review/
Allow: /properties/award-certificate/
Allow: /properties/agents/autocomplete
Allow: /properties/report/print/
Allow: /add-branch/
Allow: /report-abuse/

# Allow dynamic paths with wildcards
Allow: /find-agent/*
Allow: /view-branch/*
Allow: /rate-review/*
Allow: /properties/*
Allow: /agent/*
Allow: /supplier/*
Allow: /league-tables/*
Allow: /review/*
Allow: /house-prices/*
Allow: /free-instant-valuation/*

Allow: /review-print/*
Allow: /supplier/review/*
Allow: /properties/for-sale/*
Allow: /properties/for-rent/*
Allow: /properties/award-certificate/*
Allow: /properties/agents/*
Allow: /properties/report/print/*
Allow: /add-branch/*
Allow: /report-abuse/*

# Optional: Adding a crawl delay for specific search engines
User-agent: Bingbot

User-agent: Yandex

User-agent: Googlebot
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: bingbot
Allow: /
  
# Specify sitemap location
Sitemap: https://www.allagents.co.uk/sitemap.xml