#
# robots.txt
#
# CUSTOMISED FOR HUIS73 — do not let Composer scaffolding overwrite this file.
# It is excluded from drupal-scaffold via the "file-mapping" block in
# composer.json ("[web-root]/robots.txt": false).
#
# This file is served for every domain on this install (huis73.nl and
# ruwdenbosch.nl), so only add rules that are correct for both.
#
# Note: Drupal core's stock rules disallowed /user/login, /user/register and
# /search/ — none of which exist here. This site serves Dutch paths, and the
# stale rules left the registration and search pages fully crawlable.
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt

# Admin and core paths
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /media/oembed
Disallow: /*/media/oembed

# Search results — no index value, and every query is a distinct CDN cache
# object. This was the single most-requested path on the site.
Disallow: /search

# Account paths
Disallow: /gebruiker
Disallow: /gebruiker/login
Disallow: /gebruiker/registreren
Disallow: /gebruiker/wachtwoord
Disallow: /gebruiker/reset/
Disallow: /user/logout
Disallow: /account/
Disallow: /onderwijs-portaal/

# Commerce — cart and checkout are per-visitor and must never be crawled
Disallow: /winkelwagen
Disallow: /afronden

# Tracking parameters — duplicates of the canonical URL with a separate
# cache key.
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=

# Non-clean URL mirrors. These 301 to their clean equivalents (see
# hu_domain's IndexPhpRedirectSubscriber), but keeping the rules stops
# crawlers requesting them in the first place.
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed
Disallow: /index.php/search
Disallow: /index.php/gebruiker
Disallow: /index.php/gebruiker/
Disallow: /index.php/user/logout
Disallow: /index.php/account/
Disallow: /index.php/onderwijs-portaal/
Disallow: /index.php/winkelwagen
Disallow: /index.php/afronden

# Ignored by Google, honoured by Bing, Yandex and most smaller crawlers.
Crawl-delay: 2

Sitemap: https://www.huis73.nl/sitemap.xml
Sitemap: https://ruwdenbosch.nl/sitemap.xml