Prime-Risk-Advisory/robots.txt at main · CoreJavaPulse/Prime-Risk-Advisory · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
# Optimized robots.txt for Prime Risk Advisory
# Last updated: February 15, 2026

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /*.log$
Disallow: /*.bak$
Disallow: /cgi-bin/

# Legal pages - allow indexing despite noindex meta
Allow: /privacy-policy.html
Allow: /terms.html

# Sitemap location
Sitemap: https://primeriskadvisory.in/sitemap.xml

# Crawl-delay for respectful bots
Crawl-delay: 1

# Specific bot optimizations
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Crawl-delay: 2

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /