Manually Create ILM Policy and Template on Elasticsearch

This document contains the necessary definitions for manually creating the required log assets by Apinizer when log management needs to be handled manually.

If log management will be performed through Apinizer, these steps are not necessary as Apinizer handles them automatically.

The document provides a step-by-step explanation of all requests that need to be made to the Elasticsearch cluster. Firstly, the Index Lifecycle Management (ILM) policy is created, followed by the creation of a Template.

This way, the relevant ILM policy is linked to the Template, and an indexing definition is made in the data stream structure.

Variables

The dynamic values and explanations for the requests are listed in the table below.

Variable	Description
<ELASTICSEARCH_IP_ADDRESS>	The host information of the Elasticsearch cluster.
<ELASTICSEARCH_PORT>	The port information of the Elasticsearch cluster.

1. Creating ILM Policy

Request for Creating the ILM Policy named apinizer-log-ilm-policy-default

curl -XPUT "http://<ELASTICSEARCH_IP_ADDRESS>:<ELASTICSEARCH_PORT>/_ilm/policy/apinizer-log-ilm-policy-<INDEX_KEY>" -H 'Content-Type: application/json' -d 
'{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "30d",
            "max_size": "30gb",
            "max_docs": 15000000
          },
          "set_priority": {
            "priority": null
          }
        }
      },
      "warm": {
        "actions": {
          "readonly": {},
          "allocate": {
            "number_of_replicas": 0,
            "include": {},
            "exclude": {}
          },
          "shrink": {
            "number_of_shards": 1
          },
          "forcemerge": {
            "max_num_segments": 1
          },
          "set_priority": {
            "priority": null
          }
        }
      },
      "cold": {
        "min_age": "90d",
        "actions": {
          "set_priority": {
            "priority": null
          }
        }
      }
    }
  }
}'

CODE

2. Creating Template

Request for Creating the Template named apinizer-log-apiproxy-template-default

curl -XPUT "http://<ELASTICSEARCH_IP_ADDRESS>:<ELASTICSEARCH_PORT>/_index_template/apinizer-log-apiproxy-template-<INDEX_KEY>" -H 'Content-Type: application/json' -d
'{
  "index_patterns": ["apinizer-log-apiproxy-<INDEX_KEY>"],
  "data_stream": {},
  "template": {
    "settings": {
      "index": {
        "lifecycle": {
          "name": "apinizer-log-ilm-policy-<INDEX_KEY>"
        },
        "number_of_shards": "1",
        "number_of_replicas": "0",
        "refresh_interval":"5s"
      }
    },
    "mappings": {
      "properties": {
        "aprp": {
          "type": "keyword"
        },
        "fcrb": {
          "type": "text"
        },
        "hr1ru": {
          "type": "keyword"
        },
        "aci": {
          "type": "keyword"
        },
        "tapt": {
          "type": "integer"
        },
        "trqpt": {
          "type": "integer"
        },
        "trspt": {
          "type": "integer"
        },
        "hr1rp": {
          "type": "keyword"
        },
        "uok": {
          "ignore_above": 50,
          "type": "keyword"
        },
        "sr2t": {
          "type": "integer"
        },
        "fcrp": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "api": {
          "type": "keyword"
        },
        "appi": {
          "type": "keyword"
        },
        "appin": {
          "type": "keyword"
        },
        "fcrh": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "apn": {
          "type": "keyword"
        },
        "pet": {
          "ignore_above": 75,
          "type": "keyword"
        },
        "apr": {
          "type": "short"
        },
        "tbah": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "ei": {
          "type": "keyword"
        },
        "apmi": {
          "type": "keyword"
        },
        "hr1qs": {
          "type": "keyword"
        },
        "tbap": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "ra": {
          "type": "keyword"
        },
        "et": {
          "ignore_above": 75,
          "type": "keyword"
        },
        "tba": {
          "type": "text"
        },
        "hr1rh": {
          "type": "keyword"
        },
        "trt": {
          "type": "integer"
        },
        "rrc": {
          "type": "integer"
        },
        "rfc": {
          "type": "integer"
        },
        "sr1t": {
          "type": "integer"
        },
        "hr1ra": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword"
            }
          }
        },
        "xff": {
          "type": "keyword"
        },
        "apmn": {
          "type": "keyword"
        },
        "hr1hm": {
          "ignore_above": 50,
          "type": "keyword"
        },
        "rt": {
          "ignore_above": 7,
          "type": "keyword"
        },
        "hr1lp": {
          "type": "keyword"
        },
        "hr1ln": {
          "type": "keyword"
        },
        "hr1pi": {
          "type": "keyword"
        },
        "sc": {
          "type": "short"
        },
        "fbarh": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "tcb": {
          "type": "text"
        },
        "hr1rsi": {
          "type": "keyword"
        },
        "tch": {
          "type": "nested",
          "properties": {
            "k": {
              "type": "keyword"
            },
            "v": {
              "type": "keyword"
            }
          }
        },
        "eat": {
          "type": "keyword"
        },
        "hr1r1u": {
          "type": "keyword"
        },
        "hr1p": {
          "type": "keyword"
        },
        "hr1s": {
          "type": "keyword"
        },
        "fbarb": {
          "type": "text"
        },
        "@timestamp": {
          "format": "yyyy-MM-dd'T'HH:mm:ss.S'Z'||yyyy-MM-dd'T'HH:mm:ss.SS'Z'||yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
          "type": "date"
        },
        "ch": {
          "type": "boolean"
        },
        "hr1ce": {
          "type": "keyword"
        },
        "hr1sp": {
          "type": "keyword"
        },
        "hr1sn": {
          "type": "keyword"
        },
        "hr1la": {
          "type": "keyword"
        },
        "hr1ct": {
          "ignore_above": 50,
          "type": "keyword"
        },
        "pi": {
          "type": "keyword"
        },
        "hr1cp": {
          "type": "keyword"
        },
        "hr1cl": {
          "type": "keyword"
        }
      }
    }
  }
}'

CODE

Template Data Structure Table

Field Name	Field Description	Java Mapping Type	Elastic Search Mapping Type
@timestamp	Time Stamp	Instant	date format: yyyy-MM-dd'T'HH:mm:ss.S'Z' yyyy-MM-dd'T'HH:mm:ss.SS'Z' yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
aci	Apinizer Correlation ID	String	keyword
ei	Environment ID	String	keyword
pi	Project ID	String	keyword
api	API Proxy ID	String	keyword
appi	Application ID	String	keyword
appin	Application Name	String	keyword
apn	API Proxy Name	String	keyword
eat	API Type	Enum	keyword
apr	API Proxy Revision	Integer	short
aprp	API Proxy Relative Path	String	keyword
apmi	API Proxy Method ID	String	keyword
apmn	API Proxy Method Name	String	keyword
xff	X Forwarded For	String	keyword
hr1ra	HTTP Request Remote Address	String	text, keyword
hr1hm	HTTP Request HTTP Method	String	keyword ignore_above: 50
hr1ct	HTTP Request Content Type	String	keyword ignore_above: 50
uok	Username or Key	String	keyword ignore_above: 50
ra	Routing Address	String	keyword
rrc	Routing Retry Count	Integer	integer
rfc	Routing Failover Count	Integer	integer
trqpt	Time Request Pipeline Total	Integer	integer
trspt	Time Response Pipeline Total	Integer	integer
trt	Time Routing Total	Integer	integer
tapt	Time API Proxy Total	Integer	integer
sr1t	Size Request Total	Integer	integer
sr2t	Size Response Total	Integer	integer
ch	Cache Hit	Boolean	boolean
sc	Status Code	int	short
et	Error Type	Enum	keyword ignore_above: 75
pet	Parent Error Type	Enum	keyword ignore_above: 75
rt	Result Type	Enum	keyword ignore_above: 7
hr1pi	HTTP Request Path Info	String	keyword
hr1cp	HTTP Request Context Path	String	keyword
hr1qs	HTTP Request Query String	String	keyword
hr1ru	HTTP Request Remote User	String	keyword
hr1rsi	HTTP Request Requested SessionID	String	keyword
hr1r1u	HTTP Request Request URI	String	keyword
hr1ce	HTTP Request Character Encoding	String	keyword
hr1cl	HTTP Request Content Length	String	keyword
hr1p	HTTP Request Protocol	String	keyword
hr1s	HTTP Request Scheme	String	keyword
hr1sn	HTTP Request Server Name	String	keyword
hr1sp	HTTP Request Server Port	String	keyword
hr1rh	HTTP Request Remote Host	String	keyword
hr1rp	HTTP Request Remote Port	String	keyword
hr1ln	HTTP Request Local Name	String	keyword
hr1la	HTTP Request Local Addr	String	keyword
hr1lp	HTTP Request Local Port	String	keyword
fcrp	From Client Read Only Parameter	Map<String,String>	nested k: keyword v: keyword
fcrh	From Client Read Only Header	Map<String,String>	nested k: keyword v: keyword
fcrb	From Client Read Only Body	String	text
tbap	To Backend API Parameter	Map<String,String>	nested k: keyword v: keyword
tbah	To Backend API Header	Map<String,String>	nested k: keyword v: keyword
tba	To Backend API Body	String	text
fbarh	From Backend API Read Only Header	Map<String,String>	nested k: keyword v: keyword
fbarb	From Backend API Read Only Body	String	text
tch	To Client Header	Map<String,String>	nested k: keyword v: keyword
tcb	To Client Body	String	text

You can review the data sample formed in this data structure on the API Traffic Log Data Structure page.

3. Creating an Index and Querying

When creating an index or performing a query with a data stream, the requests are sent to the value specified in the index_patterns metadata within the template.

3.1 Creating Index and First Initialization

curl -XPOST "http://<server-host>:<server-port>/apinizer-log-apiproxy-<INDEX_KEY>/_doc" -H 'Content-Type: application/json' -d'{"@timestamp": "2023-05-31T11:40:36.376Z"}'

CODE

3.1 Sample Query

curl -XGET "http://<server-host>:<server-port>/apinizer-log-apiproxy-<INDEX_KEY>/_search" -H 'Content-Type: application/json' -d
'{
  "from": 0,
  "size": 10,
  "query": {
    "bool": {
      "filter": [
        {
          "range": {
            "@timestamp": {
              "from": "now-10m/m",
              "to": "now/m",
              "include_lower": true,
              "include_upper": true
            }
          }
        }
      ],
      "adjust_pure_negative": true,
      "boost": 1
    }
  }, 
  "sort": [
    {
      "@timestamp": {
        "order": "desc"
      }
    }
  ]
}'

CODE