1 files changed, 20 insertions(+), 2 deletions(-)

M static/robots.txt

@@ 1,2 1,20 @@
-User-agent: *
-Disallow:
+User-agent: SemrushBot
+Disallow: /
+User-agent: SemrushBot-SA
+Disallow: /
+User-agent: AhrefsBot
+Disallow: /
+User-agent: dotbot
+Disallow: /
+User-agent: rogerbot
+Disallow: /
+User-agent: BLEXBot
+Disallow: /
+User-agent: AspiegelBot
+Disallow: /
+User-agent: ZoominfoBot
+Disallow: /
+User-agent: Yandex
+Disallow: /
+User-agent: MJ12bot
+Disallow: /