Retrieve a runtime field
editRetrieve a runtime field
editUse the fields parameter on the _search API to retrieve
the values of runtime fields. Runtime fields won’t display in _source, but
the fields API works for all fields, even those that were not sent as part of
the original _source.
Define a runtime field to calculate the day of week
editFor example, the following request adds a runtime field called day_of_week.
The runtime field includes a script that calculates the day of the week based
on the value of the @timestamp field. We’ll include "dynamic":"runtime" in
the request so that new fields are added to the mapping as runtime fields.
response = client.indices.create(
index: 'my-index-000001',
body: {
mappings: {
dynamic: 'runtime',
runtime: {
day_of_week: {
type: 'keyword',
script: {
source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
},
properties: {
"@timestamp": {
type: 'date'
}
}
}
}
)
puts response
PUT my-index-000001/
{
"mappings": {
"dynamic": "runtime",
"runtime": {
"day_of_week": {
"type": "keyword",
"script": {
"source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
},
"properties": {
"@timestamp": {"type": "date"}
}
}
}
Ingest some data
editLet’s ingest some sample data, which will result in two indexed fields:
@timestamp and message.
response = client.bulk(
index: 'my-index-000001',
refresh: true,
body: [
{
index: {}
},
{
"@timestamp": '2020-06-21T15:00:01-05:00',
message: '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0'
},
{
index: {}
},
{
"@timestamp": '2020-06-21T15:00:01-05:00',
message: '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:30:17-05:00',
message: '40.135.0.0 - - [2020-04-30T14:30:17-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:30:53-05:00',
message: '232.0.0.0 - - [2020-04-30T14:30:53-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:31:12-05:00',
message: '26.1.0.0 - - [2020-04-30T14:31:12-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:31:19-05:00',
message: '247.37.0.0 - - [2020-04-30T14:31:19-05:00] "GET /french/splash_inet.html HTTP/1.0" 200 3781'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:31:27-05:00',
message: '252.0.0.0 - - [2020-04-30T14:31:27-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:31:29-05:00',
message: '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_brdl.gif HTTP/1.0" 304 0'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:31:29-05:00',
message: '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_arw.gif HTTP/1.0" 304 0'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:31:32-05:00',
message: '247.37.0.0 - - [2020-04-30T14:31:32-05:00] "GET /images/nav_bg_top.gif HTTP/1.0" 200 929'
},
{
index: {}
},
{
"@timestamp": '2020-04-30T14:31:43-05:00',
message: '247.37.0.0 - - [2020-04-30T14:31:43-05:00] "GET /french/images/nav_venue_off.gif HTTP/1.0" 304 0'
}
]
)
puts response
POST /my-index-000001/_bulk?refresh
{ "index": {}}
{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [2020-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [2020-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [2020-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [2020-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_brdl.gif HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_arw.gif HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:32-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:32-05:00] \"GET /images/nav_bg_top.gif HTTP/1.0\" 200 929"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:43-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:43-05:00] \"GET /french/images/nav_venue_off.gif HTTP/1.0\" 304 0"}
Search for the calculated day of week
editThe following request uses the search API to retrieve the day_of_week field
that the original request defined as a runtime field in the mapping. The value
for this field is calculated dynamically at query time without reindexing
documents or indexing the day_of_week field. This flexibility allows you to
modify the mapping without changing any field values.
response = client.search(
index: 'my-index-000001',
body: {
fields: [
'@timestamp',
'day_of_week'
],
_source: false
}
)
puts response
GET my-index-000001/_search
{
"fields": [
"@timestamp",
"day_of_week"
],
"_source": false
}
The previous request returns the day_of_week field for all matching documents.
We can define another runtime field called client_ip that also operates on
the message field and will further refine the query:
response = client.indices.put_mapping(
index: 'my-index-000001',
body: {
runtime: {
client_ip: {
type: 'ip',
script: {
source: 'String m = doc["message"].value; int end = m.indexOf(" "); emit(m.substring(0, end));'
}
}
}
}
)
puts response
PUT /my-index-000001/_mapping
{
"runtime": {
"client_ip": {
"type": "ip",
"script" : {
"source" : "String m = doc[\"message\"].value; int end = m.indexOf(\" \"); emit(m.substring(0, end));"
}
}
}
}
Run another query, but search for a specific IP address using the client_ip
runtime field:
GET my-index-000001/_search
{
"size": 1,
"query": {
"match": {
"client_ip": "211.11.9.0"
}
},
"fields" : ["*"]
}
This time, the response includes only two hits. The value for day_of_week
(Sunday) was calculated at query time using the runtime script defined in the
mapping, and the result includes only documents matching the 211.11.9.0 IP
address.
{
...
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "my-index-000001",
"_id" : "oWs5KXYB-XyJbifr9mrz",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2020-06-21T15:00:01-05:00",
"message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
},
"fields" : {
"@timestamp" : [
"2020-06-21T20:00:01.000Z"
],
"client_ip" : [
"211.11.9.0"
],
"message" : [
"211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
],
"day_of_week" : [
"Sunday"
]
}
}
]
}
}
Retrieve fields from related indices
editThis functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features.
The fields parameter on the _search API can also be used to retrieve fields from
the related indices via runtime fields with a type of lookup.
Fields that are retrieved by runtime fields of type lookup can be used
to enrich the hits in a search response. It’s not possible to query or aggregate
on these fields.
response = client.index(
index: 'ip_location',
refresh: true,
body: {
ip: '192.168.1.1',
country: 'Canada',
city: 'Montreal'
}
)
puts response
response = client.index(
index: 'logs',
id: 1,
refresh: true,
body: {
host: '192.168.1.1',
message: 'the first message'
}
)
puts response
response = client.index(
index: 'logs',
id: 2,
refresh: true,
body: {
host: '192.168.1.2',
message: 'the second message'
}
)
puts response
response = client.search(
index: 'logs',
body: {
runtime_mappings: {
location: {
type: 'lookup',
target_index: 'ip_location',
input_field: 'host',
target_field: 'ip',
fetch_fields: [
'country',
'city'
]
}
},
fields: [
'host',
'message',
'location'
],
_source: false
}
)
puts response
POST ip_location/_doc?refresh
{
"ip": "192.168.1.1",
"country": "Canada",
"city": "Montreal"
}
PUT logs/_doc/1?refresh
{
"host": "192.168.1.1",
"message": "the first message"
}
PUT logs/_doc/2?refresh
{
"host": "192.168.1.2",
"message": "the second message"
}
POST logs/_search
{
"runtime_mappings": {
"location": {
"type": "lookup",
"target_index": "ip_location",
"input_field": "host",
"target_field": "ip",
"fetch_fields": ["country", "city"]
}
},
"fields": [
"host",
"message",
"location"
],
"_source": false
}
|
Define a runtime field in the main search request with a type of |
|
|
The target index where the lookup query executes against |
|
|
A field on the main index whose values are used as the input values of the lookup term query |
|
|
A field on the lookup index which the lookup query searches against |
|
|
A list of fields to retrieve from the lookup index. See the |
The above search returns the country and city from the ip_location index
for each ip address of the returned search hits.
{
"took": 3,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "logs",
"_id": "1",
"_score": 1.0,
"fields": {
"host": [ "192.168.1.1" ],
"location": [
{
"city": [ "Montreal" ],
"country": [ "Canada" ]
}
],
"message": [ "the first message" ]
}
},
{
"_index": "logs",
"_id": "2",
"_score": 1.0,
"fields": {
"host": [ "192.168.1.2" ],
"message": [ "the second message" ]
}
}
]
}
}
The response of lookup fields are grouped to maintain the independence of each document from the lookup index. The lookup query for each input value is expected to match at most one document on the lookup index. If the lookup query matches more than one documents, then a random document will be selected.