# Disallow all crawlers access.
User-agent: *
Allow: /

Disallow: /404/
Disallow: /blogs/wp-admin/
Disallow: /blogs/2022/
Disallow: /blogs/category/
Disallow: /blogs/2024/
Allow: /blogs/wp-admin/admin-ajax.php
Disallow: /blackhole
Disallow: /?blackhole

# disable duggmirror
User-agent: duggmirror
Disallow: /
 
# Disallow Following crawlers access to site.

User-agent: ia_archiver-web.archive.org
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: TurnitinBot/2.1
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: SurveyBot
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: AboutUsBot
Disallow: /

User-agent: robtexbot
Disallow: /

Sitemap: https://www.thewebfactory.us/sitemap.xml
Sitemap: https://www.thewebfactory.us/blogs/sitemap_index.xml