################################# ROBOTS.TXT ###################################
#                                                                              #
# Alphabetically ordered whitelisting of legitimate web robots, which obey the #
# Robots Exclusion Standard (robots.txt). Each bot is shortly described in a   #
# comment above the (list of) user-agent(s). Comment out or delete lines which #
# contain User-agents you do not wish to allow on your website.                #
# Important: Blank lines are not allowed in the final robots.txt file!         #
# Updates can be retrieved from: https://www.ditig.com/robots-txt-template     #
#                                                                              #
# This document is licensed with a CC BY-NC-SA 4.0 license.                    #
#                                                                              #
# Last update: 2021-11-04                                                      #
#                                                                              #
################################################################################
# google.com landing page quality checks
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
# google.com app resource fetcher
User-agent: AdsBot-Google-Mobile-Apps
# bing ads bot
User-agent: adidxbot
# apple.com search engine
User-agent: Applebot
user-agent: AppleNewsBot
# bing.com international search engine
User-agent: bingbot
User-agent: BingPreview
# commoncrawl.org open repository of web crawl data
User-agent: CCBot
# duckduckgo.com international privacy search engine
User-agent: DuckDuckBot
User-agent: DuckDuckGo-Favicons-Bot
# eurip.com european search engine
User-agent: EuripBot
# facebook.com social network
User-agent: Facebot
# feedly.com feed fetcher
User-agent: Feedly
# findx.com european search engine
User-agent: Findxbot
# google.com international search engine
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Googlebot-Video
# lycos.com & hotbot.com international search engine
User-agent: Lycos
# google.com adsense bot
User-agent: Mediapartners-Google
# bing.com international search engine
User-agent: msnbot
User-agent: msnbot-media
# orange.com international search engine
User-agent: OrangeBot
# pinterest.com social networtk
User-agent: Pinterest
# yahoo.com international search engine
User-agent: Slurp
# ask.com international search engine
User-agent: Teoma
# twitter.com bot
User-agent: Twitterbot
# wotbox.com international search engine
User-agent: wotbox
# yioop.com international search engine
User-agent: YioopBot
# crawling rule(s) for above bots
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /feed/
Disallow: /comments/
Disallow: /cart/
Disallow: /checkout/
Disallow: /search/
Disallow: /my-account/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /kassa/
Disallow: /kassan/
Disallow: /varukorg/
Disallow: /mitt-konto/
Disallow: /wp-content/plugins/*.php
Disallow: /wp-content/cache/   # Optional: Test if this rule is necessary
Disallow: /wp-json/
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/
Disallow: /*add-to-cart=*
Disallow: /*orderby=*
Disallow: /*filter_*
Disallow: /*?s=
Disallow: /*?utm_*

# Allow access to essential AJAX functionality
Allow: /wp-admin/admin-ajax.php

# Allow access to specific directories and files for important resources
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-includes/js/
Allow: /*.css$
Allow: /*.js$

# disallow all other bots
User-agent: *
Disallow: /

Sitemap: https://pelagosthlm.se/sitemap_index.xml