#26 Indexing Domain-Audit logs

Author: Chris Wiechmann

docker-compose.yml

@@ -21,12 +21,13 @@ services:
     volumes:
       - type: volume
         source: filebeatdata
-        target: /usr/share/filebeat/data # Required to make sure that Filebeat remembers the last position, even if the container has been removed
-      - /etc/localtime:/etc/localtime:ro # Required to sync timezone of API-Gateway into the Docker-Container
+        target: /usr/share/filebeat/data # Required to make sure that Filebeat remembers the last position, even if the container is recreated
+      - /etc/localtime:/etc/localtime:ro # Required to sync timezone of API-Gateway into the Filebeat Docker-Container
       - ${PWD}/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
       - ${APIGATEWAY_OPENTRAFFIC_FOLDER}:/var/log/opentraffic
       - ${APIGATEWAY_TRACES_FOLDER}:/var/log/trace
       - ${APIGATEWAY_EVENTS_FOLDER}:/var/log/events
+      - ${APIGATEWAY_AUDITLOGS_FOLDER}:/var/log/audit
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
       - /var/run/docker.sock:/var/run/docker.sock
       - ${PWD}/certificates:/usr/share/filebeat/config/certificates
@@ -121,4 +122,3 @@ volumes:
 
 networks:
   elastic:
-  ingress:

elasticsearch/config/elasticsearch.yml

@@ -9,8 +9,6 @@ http.cors.allow-credentials: true
 http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
 http.cors.allow-headers: X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization. Access-Control-Allow-Headers, Accept
 
-#node.name: elasticsearch1
-#discovery.type: single-node
 bootstrap.memory_lock: true
 
 # Enable security in general

env-sample

@@ -35,6 +35,7 @@ COMPOSE_PROJECT_NAME=axway-apim-elastic
 APIGATEWAY_OPENTRAFFIC_FOLDER=/home/localuser/Axway-x.y.z/apigateway/logs/opentraffic
 APIGATEWAY_TRACES_FOLDER=/home/localuser/Axway-x.y.z/apigateway/groups/group-2/instance-1/trace
 APIGATEWAY_EVENTS_FOLDER=/home/localuser/Axway-x.y.z/apigateway/events
+APIGATEWAY_AUDITLOGS_FOLDER=/home/localuser/Axway-x.y.z/apigateway/logs
 
 # ----------------------------------------------------------------------------------------------
 # Is used by almost all services to communicate with Elasticsearch. Either to send events, 
@@ -44,7 +45,7 @@ APIGATEWAY_EVENTS_FOLDER=/home/localuser/Axway-x.y.z/apigateway/events
 # When running the ElasticSearch on a difference host (e.g. existing Elastic-Search cluster)
 # this environment variable is used to locate the ElasticSearch cluster.
 # You may provide a single host or an array of hosts.
-# Example: ELASTICSEARCH_HOSTS=["https://elasticsearch1:9200","https://elasticsearch1:9201"]
+# Example: ELASTICSEARCH_HOSTS=https://elasticsearch1:9200,https://elasticsearch2:9201
 # Used-By: Filebeat, API-Builder, Logstash
 ELASTICSEARCH_HOSTS=https://elasticsearch1:9200
 
@@ -121,7 +122,7 @@ ELASTICSEARCH_HOST3=elasticsearch3
 # Ports are different just to be able to start multiple Elasticsearch containers on one host.
 ELASTICSEARCH_HOST1_TRANSPORT=9300
 ELASTICSEARCH_HOST2_TRANSPORT=9301
-ELASTICSEARCH_HOST3_TRANSPORT=9301
+ELASTICSEARCH_HOST3_TRANSPORT=9302
 
 ################################################################################################
 #                            Optional / Advanced parameters

logstash/config/pipelines.yml

@@ -23,4 +23,9 @@
 - pipeline.id: Events
   path.config: "pipelines/EventsPipeline.conf"
   pipeline.workers: 1
+  pipeline.ordered: false
+
+- pipeline.id: DomainAudit
+  path.config: "pipelines/DomainAuditPipeline.conf"
+  pipeline.workers: 1
   pipeline.ordered: false

logstash/index_templates/domainaudit_index_template.json

@@ -0,0 +1,51 @@
+{
+    "version": "1.0.0",
+    "index_patterns": [
+        "apigw-domainaudit"
+    ],
+    "settings": {
+        "number_of_shards": 1, 
+        "number_of_replicas": 1,
+        "index": {
+            "codec": "best_compression"
+        }
+    },
+    "mappings": {
+        "dynamic": false,
+        "properties": {
+            "@timestamp": {
+                "type" : "date"
+            },
+            "message": {
+                "type": "text"
+            },
+            "additionalInfo": {
+                "type": "text"
+            },
+            "eventId": {
+                "type": "integer"
+            },
+            "user": {
+                "type": "keyword"
+            },
+            "outcome": {
+                "type": "keyword"
+            },
+            "metaData.serviceID": {
+                "type": "keyword"
+            },
+            "metaData.clientAddr": {
+                "type": "keyword"
+            },
+            "metaData.Referer": {
+                "type": "keyword"
+            },
+            "metaData.Host": {
+                "type": "keyword"
+            },
+            "metaData.requestURI": {
+                "type": "text"
+            }
+        }
+    }
+}

logstash/index_templates/monitoring_event_index_template.json

@@ -1,4 +1,5 @@
 {
+    "version": 1,
     "index_patterns": [
         "apigw-monitoring-*"
     ],

logstash/index_templates/trace_messages_index_template.json

@@ -1,4 +1,5 @@
 {
+    "version": 1,
     "index_patterns": [
         "apigw-trace-*", 
         "apigw-traffic-trace-*"

logstash/index_templates/traffic_details_index_template.json

@@ -1,4 +1,5 @@
 {
+    "version": 1,
     "index_patterns": [
         "apigw-traffic-details-*"
     ],

logstash/index_templates/traffic_summary_index_template.json

@@ -1,4 +1,5 @@
 {
+    "version": 1,
     "index_patterns": [
         "apigw-traffic-summary-*"
     ],

logstash/pipelines/BeatsInputPipeline.conf

@@ -25,6 +25,11 @@ output {
         send_to => "Events" 
         id => "Events"
       }
+  } else if [logtype] == "domainaudit" {
+      pipeline { 
+        send_to => "DomainAudit" 
+        id => "DomainAudit"
+      }
   } else {
     elasticsearch {
       hosts => "${ELASTICSEARCH_HOSTS}"