md/robots.txt at master · peterstone649/md · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# Robots.txt for MD Repository
# https://github.com/peterstone649/md

User-agent: *
Allow: /

# Allow all major search engines to crawl
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Disallow crawling of generated/output directories (except HTML exports)
Disallow: /__pycache__/
Disallow: /.pytest_cache/
Disallow: /.git/

# Allow crawling of documentation
Allow: /MODEL_for_framework/
Allow: /MODEL_for_STKHLD_AI_COLLAB/
Allow: /BOOK/
Allow: /transl/
Allow: /out/html/
Allow: /README.md
Allow: /CONTRIBUTING.md

# Sitemap location
Sitemap: https://github.com/peterstone649/md/blob/master/sitemap.xml

# Crawl delay (be respectful to GitHub)
Crawl-delay: 1