From db4a48bb98b6a6d9685d07e023bbb0a71ddde617 Mon Sep 17 00:00:00 2001
From: "J.-S. Caux" <J.S.Caux@uva.nl>
Date: Sun, 18 Apr 2021 10:23:40 +0200
Subject: [PATCH] Add crawl-delay to ahrefs bot

---
 robots.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/robots.txt b/robots.txt
index 152a53455..fb12cb233 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,2 +1,5 @@
+User-agent: AhrefsBot
+Crawl-Delay: 10
+
 User-agent: BLEXBot
 Disallow: /
-- 
GitLab