# robots.txt
# Server: mittelstandcafe.de
# last change: 2025-03-31

User-agent: magpie-crawler 
Disallow: /
User-agent: 008
Disallow: /
User-agent: ccbot Disallow: /
User-agent: mj12bot Disallow: /
User-agent: baiduspider Disallow: /
User-agent: baiduspider-image Disallow: /
User-agent: baiduspider-video Disallow: /
User-agent: baiduspider-news Disallow: /
User-agent: baiduspider-favo Disallow: /
User-agent: baiduspider-cpro Disallow: /
User-agent: baiduspider-ads Disallow: /
User-agent: SemrushBot Disallow: /
User-agent: SemruchBot-SA Disallow: /
User-agent: ahrefsbot Disallow: /
User-agent: PetalBot Disallow: /
User-agent: dotbot Disallow: /
User-agent: Yandex Disallow: /
User-agent: VocusBot Disallow: /
User-agent: suggybot Disallow: /
User-agent: ShopWiki Disallow: /
User-agent: emefgebot Disallow: /
User-agent: GermCrawler Disallow: /
User-agent: WBSearchBot Disallow: /
User-agent: discobot Disallow: /
User-agent: Exabot Disallow: /
User-agent: YoudaoBot Disallow: /
User-agent: NaverBot Disallow: /
User-agent: Spinn3r Disallow: /
User-agent: MLBot Disallow: /
User-agent: YioopBot Disallow: /
User-agent: Yeti Disallow: /
User-agent: moget Disallow: /
User-agent: netseer Disallow: /
User-agent: ichiro Disallow: /
User-agent: JikeSpider Disallow: /
User-agent: MaxPointCrawler/Nutch-1.1 Disallow: /
User-agent: CrystalSemanticsBot Disallow: /
User-agent: TurnitinBot Disallow: /
User-agent: yacybot Disallow: /
User-agent: NerdByNature Disallow: /
User-agent: acoon Disallow: /
User-agent: Ezooms Disallow: /
User-agent: Java Disallow: /
User-agent: libwww-perl Disallow: /
User-agent: niki-bot Disallow: /
User-agent: updown_tester Disallow: /

User-agent: looksmart Disallow: /
User-agent: WebZip Disallow: /
User-agent: larbin Disallow: /
User-agent: b2w/0.1 Disallow: /
User-agent: psbot Disallow: /
User-agent: Python-urllib Disallow: /
User-agent: NetMechanic Disallow: /
User-agent: URL_Spider_Pro Disallow: /
User-agent: CherryPicker Disallow: /
User-agent: EmailCollector Disallow: /
User-agent: EmailSiphon Disallow: /
User-agent: WebBandit Disallow: /
User-agent: EmailWolf Disallow: /
User-agent: ExtractorPro Disallow: /
User-agent: CopyRightCheck Disallow: /
User-agent: Crescent Disallow: /
User-agent: SiteSnagger Disallow: /
User-agent: ProWebWalker Disallow: /
User-agent: CheeseBot Disallow: /
User-agent: LNSpiderguy Disallow: /
User-agent: ia_archiver Disallow: /
User-agent: ia_archiver/1.6 Disallow: /
User-agent: Teleport Disallow: /
User-agent: TeleportPro Disallow: /
User-agent: MIIxpc Disallow: /
User-agent: Telesoft Disallow: /
User-agent: Website Quester Disallow: /
User-agent: moget/2.1 Disallow: /
User-agent: WebZip/4.0 Disallow: /
User-agent: WebStripper Disallow: /
User-agent: WebSauger Disallow: /
User-agent: WebCopier Disallow: /
User-agent: NetAnts Disallow: /
User-agent: Mister PiX Disallow: /
User-agent: WebAuto Disallow: /
User-agent: TheNomad Disallow: /
User-agent: WWW-Collector-E Disallow: /
User-agent: RMA Disallow: /
User-agent: libWeb/clsHTTP Disallow: /
User-agent: asterias Disallow: /
User-agent: httplib Disallow: /
User-agent: turingos Disallow: /
User-agent: spanner Disallow: /
User-agent: InfoNaviRobot Disallow: /
User-agent: Harvest/1.5 Disallow: /
User-agent: Bullseye/1.0 Disallow: /
User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95) Disallow: /
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: /
User-agent: CherryPickerSE/1.0 Disallow: /
User-agent: CherryPickerElite/1.0 Disallow: /
User-agent: WebBandit/3.50 Disallow: /
User-agent: NICErsPRO Disallow: /
User-agent: Microsoft URL Control - 5.01.4511 Disallow: /
User-agent: DittoSpyder Disallow: /
User-agent: Foobot Disallow: /
User-agent: WebmasterWorldForumBot Disallow: /
User-agent: SpankBot Disallow: /
User-agent: BotALot Disallow: /
User-agent: lwp-trivial/1.34 Disallow: /
User-agent: lwp-trivial Disallow: /
User-agent: BunnySlippers Disallow: /
User-agent: Microsoft URL Control - 6.00.8169 Disallow: /
User-agent: URLy Warning Disallow: /
User-agent: Wget/1.6 Disallow: /
User-agent: Wget/1.5.3 Disallow: /
User-agent: Wget Disallow: /
User-agent: LinkWalker Disallow: /
User-agent: cosmos Disallow: /
User-agent: moget Disallow: /
User-agent: hloader Disallow: /
User-agent: humanlinks Disallow: /
User-agent: LinkextractorPro Disallow: /
User-agent: Offline Explorer Disallow: /
User-agent: Mata Hari Disallow: /
User-agent: LexiBot Disallow: /
User-agent: Web Image Collector Disallow: /
User-agent: The Intraformant Disallow: /
User-agent: True_Robot/1.0 Disallow: /
User-agent: True_Robot Disallow: /
User-agent: BlowFish/1.0 Disallow: /
User-agent: JennyBot Disallow: /
User-agent: MIIxpc/4.2 Disallow: /
User-agent: BuiltBotTough Disallow: /
User-agent: ProPowerBot/2.14 Disallow: /
User-agent: BackDoorBot/1.0 Disallow: /
User-agent: toCrawl/UrlDispatcher Disallow: /
User-agent: WebEnhancer Disallow: /
User-agent: suzuran Disallow: /
User-agent: VCI WebViewer VCI WebViewer Win32 Disallow: /
User-agent: VCI Disallow: /
User-agent: Szukacz/1.4 Disallow: /
User-agent: QueryN Metasearch Disallow: /
User-agent: Openfind data gathere Disallow: /
User-agent: Openfind Disallow: /
User-agent: Xenu's Link Sleuth 1.1c Disallow: /
User-agent: Xenu's Disallow: /
User-agent: Zeus Disallow: /
User-agent: RepoMonkey Bait & Tackle/v1.01 Disallow: /
User-agent: RepoMonkey Disallow: /
User-agent: Microsoft URL Control Disallow: /
User-agent: Openbot Disallow: /
User-agent: URL Control Disallow: /
User-agent: Zeus Link Scout Disallow: /
User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: /
User-agent: Webster Pro Disallow: /
User-agent: EroCrawler Disallow: /
User-agent: LinkScan/8.1a Unix Disallow: /
User-agent: Keyword Recherche Density/0.9 Disallow: /
User-agent: Kenjin Spider Disallow: /
User-agent: Iron33/1.0.2 Disallow: /
User-agent: Bookmark search tool Disallow: /
User-agent: GetRight/4.2 Disallow: /
User-agent: FairAd Client Disallow: /
User-agent: Gaisbot Disallow: /
User-agent: Aqua_Products Disallow: /
User-agent: Radiation Retriever 1.1 Disallow: /
User-agent: Flaming AttackBot Disallow: /
User-agent: Oracle Ultra Search Disallow: /
User-agent: MSIECrawler Disallow: /
User-agent: PerMan Disallow: /
User-agent: searchpreview Disallow: /

# Performance Testing
User-agent: 200PleaseBot Disallow: /
User-agent: ApacheBench Disallow: /
User-agent: LoadTimeBot Disallow: /

# Harvester & Marketing
User-agent: CareerBot Disallow: /
User-agent: GrapeshotCrawler Disallow: /
User-agent: iCjobs Disallow: /
User-agent: magpie-crawler Disallow: /
User-agent: proximic Disallow: /
User-agent: QuerySeekerSpider Disallow: /
User-agent: Vagabondo Disallow: /
User-agent: WBSearchBot Disallow: /

# Backlinks & Links
User-agent: AhrefsBot Disallow: /
User-agent: BacklinkCrawler Disallow: /
User-agent: dlcbot Disallow: /
User-agent: spbot Disallow: /

# SEO
User-agent: AiHitBot Disallow: /
User-agent: BLEXBot Disallow: /
User-agent: Cliqzbot Disallow: /
User-agent: DotBot Disallow: /
User-agent: linkdexbot Disallow: /
User-agent: MJ12bot Disallow: /
User-agent: SEOkicks-Robot Disallow: /
User-agent: SearchmetricsBot Disallow: /
User-agent: SISTRIX Disallow: /

# Zentrale
User-agent: MJ12bot	
Disallow: /	
User-agent: SemrushBot	
Disallow: /	
User-agent: jobs.de-Robot	
Disallow: /	
User-agent: spbot	
Disallow: /	
User-agent: JobboerseBot	
Disallow: /	
User-agent: trendictionbot	
Disallow: /	
User-agent: Cliqzbot	
Disallow: /	
User-agent: SeznamBot	
Disallow: /	
User-agent: Exabot	
Disallow: /	
User-agent: Mediatoolkitbot	
Disallow: /	
User-agent: Mail.RU_Bot	
Disallow: /	
User-agent: Wotbox	
Disallow: /	
User-agent: AlphaBot	
Disallow: /	
User-agent: Baiduspider	
Disallow: /	
User-agent: YandexMobileBot	
Disallow: /	
User-agent: SMTBot	
Disallow: /	
User-agent: rogerbot	
Disallow: /	
User-agent: YandexImages	
Disallow: /	
User-agent: Yeti	
Disallow: /	
User-agent: linkdexbot	
Disallow: /	
User-agent: SEOkicks-Robot	
Disallow: /	
User-agent: YaK	
Disallow: /	
User-agent: MojeekBot	
Disallow: /	
User-agent: wonderbot	
Disallow: /	
User-agent: YandexAntivirus	
Disallow: /	
User-agent: vebidoobot	
Disallow: /	
User-agent: Pinterestbot	
Disallow: /	
User-agent: TweetmemeBot	
Disallow: /	
User-agent: changedetection	
Disallow: /	
User-agent: SafeDNSBot	
Disallow: /	
User-agent: yacybot	
Disallow: /	
User-agent: Buck	
Disallow: /	
User-agent: SemrushBot-SA	
Disallow: /	
User-agent: SemrushBot-BA	
Disallow: /	
User-agent: SemrushBot-BM	
Disallow: /	
User-agent: Grobbot	
Disallow: /	
User-agent: Baiduspider-render	
Disallow: /	
User-agent: AhrefsBot	
Disallow: /	
User-agent: Flamingo_SearchEngine	
Disallow: /	
User-agent: MegaIndex.ru	
Disallow: /	
User-agent: Mega-Index	
Disallow: /

User-agent: *
Disallow: /wp-json 
Disallow: /admin
Disallow: /albums/
Disallow: /blocks/
Disallow: /images/
Disallow: /includes/
Disallow: /install/
Disallow: /language/
Disallow: /modules/
Disallow: /spaw/
Disallow: /themes/
Disallow: /upgrade/
Disallow: /themecache/
Disallow: /dynadata/
Disallow: /setup/
Disallow: /mobile/
Disallow: /bottrap/
Disallow: /cgi-bin/suite/

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /search/

#Sitemaps
Sitemap: https://www.mittelstandcafe.de/mc-sitemap.xml