« Override field values at query time Index a runtime field »
Elastic Docs ›Elasticsearch Guide [8.16] ›Mapping ›Runtime fields

Retrieve a runtime field

edit

Retrieve a runtime field

edit

Use the fields parameter on the _search API to retrieve the values of runtime fields. Runtime fields won’t display in _source, but the fields API works for all fields, even those that were not sent as part of the original _source.

Define a runtime field to calculate the day of week

edit

For example, the following request adds a runtime field called day_of_week. The runtime field includes a script that calculates the day of the week based on the value of the @timestamp field. We’ll include "dynamic":"runtime" in the request so that new fields are added to the mapping as runtime fields.

resp = client.indices.create(
    index="my-index-000001",
    mappings={
        "dynamic": "runtime",
        "runtime": {
            "day_of_week": {
                "type": "keyword",
                "script": {
                    "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
                }
            }
        },
        "properties": {
            "@timestamp": {
                "type": "date"
            }
        }
    },
)
print(resp)
const response = await client.indices.create({
  index: "my-index-000001",
  mappings: {
    dynamic: "runtime",
    runtime: {
      day_of_week: {
        type: "keyword",
        script: {
          source:
            "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))",
        },
      },
    },
    properties: {
      "@timestamp": {
        type: "date",
      },
    },
  },
});
console.log(response);
PUT my-index-000001/
{
  "mappings": {
    "dynamic": "runtime",
    "runtime": {
      "day_of_week": {
        "type": "keyword",
        "script": {
          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
        }
      }
    },
    "properties": {
      "@timestamp": {"type": "date"}
    }
  }
}

Ingest some data

edit

Let’s ingest some sample data, which will result in two indexed fields: @timestamp and message.

resp = client.bulk(
    index="my-index-000001",
    refresh=True,
    operations=[
        {
            "index": {}
        },
        {
            "@timestamp": "2020-06-21T15:00:01-05:00",
            "message": "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-06-21T15:00:01-05:00",
            "message": "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:30:17-05:00",
            "message": "40.135.0.0 - - [2020-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:30:53-05:00",
            "message": "232.0.0.0 - - [2020-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:12-05:00",
            "message": "26.1.0.0 - - [2020-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:19-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:27-05:00",
            "message": "252.0.0.0 - - [2020-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:29-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_brdl.gif HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:29-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_arw.gif HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:32-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:32-05:00] \"GET /images/nav_bg_top.gif HTTP/1.0\" 200 929"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:43-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:43-05:00] \"GET /french/images/nav_venue_off.gif HTTP/1.0\" 304 0"
        }
    ],
)
print(resp)
response = client.bulk(
  index: 'my-index-000001',
  refresh: true,
  body: [
    {
      index: {}
    },
    {
      "@timestamp": '2020-06-21T15:00:01-05:00',
      message: '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-06-21T15:00:01-05:00',
      message: '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:30:17-05:00',
      message: '40.135.0.0 - - [2020-04-30T14:30:17-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:30:53-05:00',
      message: '232.0.0.0 - - [2020-04-30T14:30:53-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:12-05:00',
      message: '26.1.0.0 - - [2020-04-30T14:31:12-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:19-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:19-05:00] "GET /french/splash_inet.html HTTP/1.0" 200 3781'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:27-05:00',
      message: '252.0.0.0 - - [2020-04-30T14:31:27-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:29-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_brdl.gif HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:29-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_arw.gif HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:32-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:32-05:00] "GET /images/nav_bg_top.gif HTTP/1.0" 200 929'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:43-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:43-05:00] "GET /french/images/nav_venue_off.gif HTTP/1.0" 304 0'
    }
  ]
)
puts response
const response = await client.bulk({
  index: "my-index-000001",
  refresh: "true",
  operations: [
    {
      index: {},
    },
    {
      "@timestamp": "2020-06-21T15:00:01-05:00",
      message:
        '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-06-21T15:00:01-05:00",
      message:
        '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:30:17-05:00",
      message:
        '40.135.0.0 - - [2020-04-30T14:30:17-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:30:53-05:00",
      message:
        '232.0.0.0 - - [2020-04-30T14:30:53-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:12-05:00",
      message:
        '26.1.0.0 - - [2020-04-30T14:31:12-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:19-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:19-05:00] "GET /french/splash_inet.html HTTP/1.0" 200 3781',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:27-05:00",
      message:
        '252.0.0.0 - - [2020-04-30T14:31:27-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:29-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_brdl.gif HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:29-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_arw.gif HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:32-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:32-05:00] "GET /images/nav_bg_top.gif HTTP/1.0" 200 929',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:43-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:43-05:00] "GET /french/images/nav_venue_off.gif HTTP/1.0" 304 0',
    },
  ],
});
console.log(response);
POST /my-index-000001/_bulk?refresh
{ "index": {}}
{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [2020-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [2020-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [2020-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [2020-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_brdl.gif HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_arw.gif HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:32-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:32-05:00] \"GET /images/nav_bg_top.gif HTTP/1.0\" 200 929"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:43-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:43-05:00] \"GET /french/images/nav_venue_off.gif HTTP/1.0\" 304 0"}

Search for the calculated day of week

edit

The following request uses the search API to retrieve the day_of_week field that the original request defined as a runtime field in the mapping. The value for this field is calculated dynamically at query time without reindexing documents or indexing the day_of_week field. This flexibility allows you to modify the mapping without changing any field values.

resp = client.search(
    index="my-index-000001",
    fields=[
        "@timestamp",
        "day_of_week"
    ],
    source=False,
)
print(resp)
response = client.search(
  index: 'my-index-000001',
  body: {
    fields: [
      '@timestamp',
      'day_of_week'
    ],
    _source: false
  }
)
puts response
const response = await client.search({
  index: "my-index-000001",
  fields: ["@timestamp", "day_of_week"],
  _source: false,
});
console.log(response);
GET my-index-000001/_search
{
  "fields": [
    "@timestamp",
    "day_of_week"
  ],
  "_source": false
}

The previous request returns the day_of_week field for all matching documents. We can define another runtime field called client_ip that also operates on the message field and will further refine the query:

resp = client.indices.put_mapping(
    index="my-index-000001",
    runtime={
        "client_ip": {
            "type": "ip",
            "script": {
                "source": "String m = doc[\"message\"].value; int end = m.indexOf(\" \"); emit(m.substring(0, end));"
            }
        }
    },
)
print(resp)
response = client.indices.put_mapping(
  index: 'my-index-000001',
  body: {
    runtime: {
      client_ip: {
        type: 'ip',
        script: {
          source: 'String m = doc["message"].value; int end = m.indexOf(" "); emit(m.substring(0, end));'
        }
      }
    }
  }
)
puts response
const response = await client.indices.putMapping({
  index: "my-index-000001",
  runtime: {
    client_ip: {
      type: "ip",
      script: {
        source:
          'String m = doc["message"].value; int end = m.indexOf(" "); emit(m.substring(0, end));',
      },
    },
  },
});
console.log(response);
PUT /my-index-000001/_mapping
{
  "runtime": {
    "client_ip": {
      "type": "ip",
      "script" : {
      "source" : "String m = doc[\"message\"].value; int end = m.indexOf(\" \"); emit(m.substring(0, end));"
      }
    }
  }
}

Run another query, but search for a specific IP address using the client_ip runtime field:

resp = client.search(
    index="my-index-000001",
    size=1,
    query={
        "match": {
            "client_ip": "211.11.9.0"
        }
    },
    fields=[
        "*"
    ],
)
print(resp)
const response = await client.search({
  index: "my-index-000001",
  size: 1,
  query: {
    match: {
      client_ip: "211.11.9.0",
    },
  },
  fields: ["*"],
});
console.log(response);
GET my-index-000001/_search
{
  "size": 1,
  "query": {
    "match": {
      "client_ip": "211.11.9.0"
    }
  },
  "fields" : ["*"]
}

This time, the response includes only two hits. The value for day_of_week (Sunday) was calculated at query time using the runtime script defined in the mapping, and the result includes only documents matching the 211.11.9.0 IP address.

{
  ...
  "hits" : {
    "total" : {
      "value" : 2,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "my-index-000001",
        "_id" : "oWs5KXYB-XyJbifr9mrz",
        "_score" : 1.0,
        "_source" : {
          "@timestamp" : "2020-06-21T15:00:01-05:00",
          "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
        },
        "fields" : {
          "@timestamp" : [
            "2020-06-21T20:00:01.000Z"
          ],
          "client_ip" : [
            "211.11.9.0"
          ],
          "message" : [
            "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
          ],
          "day_of_week" : [
            "Sunday"
          ]
        }
      }
    ]
  }
}

Retrieve fields from related indices

edit

The fields parameter on the _search API can also be used to retrieve fields from the related indices via runtime fields with a type of lookup.

Fields that are retrieved by runtime fields of type lookup can be used to enrich the hits in a search response. It’s not possible to query or aggregate on these fields.

resp = client.index(
    index="ip_location",
    refresh=True,
    document={
        "ip": "192.168.1.1",
        "country": "Canada",
        "city": "Montreal"
    },
)
print(resp)

resp1 = client.index(
    index="logs",
    id="1",
    refresh=True,
    document={
        "host": "192.168.1.1",
        "message": "the first message"
    },
)
print(resp1)

resp2 = client.index(
    index="logs",
    id="2",
    refresh=True,
    document={
        "host": "192.168.1.2",
        "message": "the second message"
    },
)
print(resp2)

resp3 = client.search(
    index="logs",
    runtime_mappings={
        "location": {
            "type": "lookup",
            "target_index": "ip_location",
            "input_field": "host",
            "target_field": "ip",
            "fetch_fields": [
                "country",
                "city"
            ]
        }
    },
    fields=[
        "host",
        "message",
        "location"
    ],
    source=False,
)
print(resp3)
response = client.index(
  index: 'ip_location',
  refresh: true,
  body: {
    ip: '192.168.1.1',
    country: 'Canada',
    city: 'Montreal'
  }
)
puts response

response = client.index(
  index: 'logs',
  id: 1,
  refresh: true,
  body: {
    host: '192.168.1.1',
    message: 'the first message'
  }
)
puts response

response = client.index(
  index: 'logs',
  id: 2,
  refresh: true,
  body: {
    host: '192.168.1.2',
    message: 'the second message'
  }
)
puts response

response = client.search(
  index: 'logs',
  body: {
    runtime_mappings: {
      location: {
        type: 'lookup',
        target_index: 'ip_location',
        input_field: 'host',
        target_field: 'ip',
        fetch_fields: [
          'country',
          'city'
        ]
      }
    },
    fields: [
      'host',
      'message',
      'location'
    ],
    _source: false
  }
)
puts response
const response = await client.index({
  index: "ip_location",
  refresh: "true",
  document: {
    ip: "192.168.1.1",
    country: "Canada",
    city: "Montreal",
  },
});
console.log(response);

const response1 = await client.index({
  index: "logs",
  id: 1,
  refresh: "true",
  document: {
    host: "192.168.1.1",
    message: "the first message",
  },
});
console.log(response1);

const response2 = await client.index({
  index: "logs",
  id: 2,
  refresh: "true",
  document: {
    host: "192.168.1.2",
    message: "the second message",
  },
});
console.log(response2);

const response3 = await client.search({
  index: "logs",
  runtime_mappings: {
    location: {
      type: "lookup",
      target_index: "ip_location",
      input_field: "host",
      target_field: "ip",
      fetch_fields: ["country", "city"],
    },
  },
  fields: ["host", "message", "location"],
  _source: false,
});
console.log(response3);
POST ip_location/_doc?refresh
{
  "ip": "192.168.1.1",
  "country": "Canada",
  "city": "Montreal"
}

PUT logs/_doc/1?refresh
{
  "host": "192.168.1.1",
  "message": "the first message"
}

PUT logs/_doc/2?refresh
{
  "host": "192.168.1.2",
  "message": "the second message"
}

POST logs/_search
{
  "runtime_mappings": {
    "location": {
        "type": "lookup", 
        "target_index": "ip_location", 
        "input_field": "host", 
        "target_field": "ip", 
        "fetch_fields": ["country", "city"] 
    }
  },
  "fields": [
    "host",
    "message",
    "location"
  ],
  "_source": false
}

Define a runtime field in the main search request with a type of lookup that retrieves fields from the target index using the term queries.

The target index where the lookup query executes against

A field on the main index whose values are used as the input values of the lookup term query

A field on the lookup index which the lookup query searches against

A list of fields to retrieve from the lookup index. See the fields parameter of a search request.

The above search returns the country and city from the ip_location index for each ip address of the returned search hits.

{
  "took": 3,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 2,
      "relation": "eq"
    },
    "max_score": 1.0,
    "hits": [
      {
        "_index": "logs",
        "_id": "1",
        "_score": 1.0,
        "fields": {
          "host": [ "192.168.1.1" ],
          "location": [
            {
              "city": [ "Montreal" ],
              "country": [ "Canada" ]
            }
          ],
          "message": [ "the first message" ]
        }
      },
      {
        "_index": "logs",
        "_id": "2",
        "_score": 1.0,
        "fields": {
          "host": [ "192.168.1.2" ],
          "message": [ "the second message" ]
        }
      }
    ]
  }
}

The response of lookup fields are grouped to maintain the independence of each document from the lookup index. The lookup query for each input value is expected to match at most one document on the lookup index. If the lookup query matches more than one documents, then a random document will be selected.

« Override field values at query time Index a runtime field »