#User-agent: * #Crawl-delay: 20 User-agent: Yandex Crawl-delay: 2 # sets a 2-second timeout User-agent: GPTBot Disallow: /