# robots.txt — www.omegamoulding.com / www.omegamoulding.ca
# v2 (tightened), live since 2026-09-13. Replaces the 2016 "Allow: /" file.
#
# Principle: crawl the catalogue (PDPs, category landings, press, images, assets) for the engines
# that send US/CA customers; nothing else. Never the session/cart/account layer, never side-effect
# endpoints, never AJAX fragments, never the infinite facet space.

User-agent: *

# --- session / commerce (state-bearing; crawler hits create sessions and cart work) ---
Disallow: /orders/
Disallow: /cart/
Disallow: /account
Disallow: /login
Disallow: /login2/
Disallow: /logout
Disallow: /register
Disallow: /signin
Disallow: /signup
Disallow: /wishlist/
Disallow: /resources/favorites
Disallow: /redeemItems/
Disallow: /product/order/

# --- side-effect + backend endpoints (counters, persistence, mail, auth) ---
Disallow: /products/incViewed
Disallow: /products/incAcquired
Disallow: /products/substitute
Disallow: /products/substitutes
Disallow: /services/
Disallow: /components/
Disallow: /api/
Disallow: /data/
Disallow: /urgentAlerts
Disallow: /sitekick/

# --- AJAX fragments / test pages (not real pages, duplicate or empty content) ---
Disallow: /product/nonMouldingPopup
Disallow: /product/detail_c
Disallow: /product/prdtest
Disallow: /product/zoomtest
Disallow: /search/
Disallow: /search5
Disallow: /search8
Disallow: /search0
Disallow: /search110
Disallow: /searchtarget
Disallow: /searchstat
Disallow: /searchdev
Disallow: /products/mouldings/colors
Disallow: /products/mouldings/sizes
Disallow: /srinivas/
Disallow: /fsevt/
Disallow: /errorhandlers/

# --- deep search facets = infinite URL space. Multi-condition (%2c / ,), colour, width, brand,
#     status, collection and cat filters are blocked; category landings (…&landing=true) win via the
#     longer Allow; plain /search?q=word stays open. ---
Disallow: /search?*%2c
Disallow: /search?*%2C
Disallow: /search?*,
Disallow: /search?*color=
Disallow: /search?*width=
Disallow: /search?*brand=
Disallow: /search?*collection=
Disallow: /search?*status=
Disallow: /search?*cat1=
Disallow: /search?*cat2=
Disallow: /search?*&q=$
Allow: /search?*landing=true

# --- legacy ASP.NET URLs: all soft-404 (200 + "404 | Omega") ---
Disallow: /*.aspx$
Disallow: /*.aspx?
Disallow: /ssf/

# --- explicitly open (search-engine rendering needs these) ---
Allow: /product/detail/
Allow: /products/
Allow: /search$
Allow: /search?q=
Allow: /pressroom/
Allow: /content/
Allow: /js/
Allow: /css/
Allow: /img/
Allow: /fonts/
Allow: /components/*.css
Allow: /components/*.js

# --- Bing + Apple: same rules at 1 request / 10 s (Google ignores Crawl-delay; these two honour it).
#     A named group does not inherit from "*", so the rules are repeated. ---
User-agent: bingbot
User-agent: Applebot
Crawl-delay: 10
Disallow: /orders/
Disallow: /cart/
Disallow: /account
Disallow: /login
Disallow: /login2/
Disallow: /logout
Disallow: /register
Disallow: /signin
Disallow: /signup
Disallow: /wishlist/
Disallow: /resources/favorites
Disallow: /redeemItems/
Disallow: /product/order/
Disallow: /products/incViewed
Disallow: /products/incAcquired
Disallow: /products/substitute
Disallow: /products/substitutes
Disallow: /services/
Disallow: /components/
Disallow: /api/
Disallow: /data/
Disallow: /urgentAlerts
Disallow: /sitekick/
Disallow: /product/nonMouldingPopup
Disallow: /product/detail_c
Disallow: /product/prdtest
Disallow: /product/zoomtest
Disallow: /search/
Disallow: /search5
Disallow: /search8
Disallow: /search0
Disallow: /search110
Disallow: /searchtarget
Disallow: /searchstat
Disallow: /searchdev
Disallow: /products/mouldings/colors
Disallow: /products/mouldings/sizes
Disallow: /srinivas/
Disallow: /fsevt/
Disallow: /errorhandlers/
Disallow: /search?*%2c
Disallow: /search?*%2C
Disallow: /search?*,
Disallow: /search?*color=
Disallow: /search?*width=
Disallow: /search?*brand=
Disallow: /search?*collection=
Disallow: /search?*status=
Disallow: /search?*cat1=
Disallow: /search?*cat2=
Disallow: /search?*&q=$
Allow: /search?*landing=true
Disallow: /*.aspx$
Disallow: /*.aspx?
Disallow: /ssf/
Allow: /components/*.css
Allow: /components/*.js

# --- no value for a US/CA B2B catalogue: CN/RU search engines (Baidu alone = 25% of all crawler
#     traffic here), scrapers, SEO-tool crawlers. Bytespider ignores Crawl-delay; only a block works. ---
User-agent: Baiduspider
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: YandexBot
User-agent: Bytespider
User-agent: PetalBot
User-agent: UsefulEconomyBot
User-agent: Reflectionbot
User-agent: SiteAuditBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: Barkrowler
User-agent: ImagesiftBot
User-agent: Timpibot
User-agent: YouBot
Disallow: /

# --- AI training crawlers: blocked. AI *search/answer* agents (OAI-SearchBot, ChatGPT-User,
#     PerplexityBot, Claude-User, DuckAssistBot) stay under "*" so the brand remains citable.
#     Google-Extended / Applebot-Extended affect training only, not Search / Siri. ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: omgili
Disallow: /

Sitemap: https://www.omegamoulding.com/sitemap.xml