docs/robots.txt at development · jandevriesmendix/docs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
---
layout: null
permalink: robots.txt
---
# filter pages and documents for the noindex key
User-agent: *
Disallow: /mappings/
Disallow: /json/
Sitemap: https://docs.mendix.com/sitemap.xml
# robotstxt.org - if _config production variable is false robots will be disallowed.
{% assign noindexPages = site.pages | where: 'sitemap', false %}
{% assign noindexDocuments = site.documents | where: 'sitemap', false %}
{% if jekyll.environment != "production" %}
  Disallow: /
{% else %}
  {% if noindexDocuments or noindexPages %}
    {% for node in noindexPages %}
      Disallow: {{ node.url }}
    {% endfor %}
    {% for node in noindexDocuments %}
      Disallow: {{ node.url }}
    {% endfor %}
  {% else %}
    Disallow:
  {% endif %}
{% endif %}