Error Indices

edit

Errors are stored in separate indices of the format apm-[version]-error-[date].

Example documents

edit

See how error documents can look like when indexed in Elasticsearch:

{
    "events": [
        {
            "@timestamp": "2017-05-09T15:04:05.999999Z",
            "agent": {
                "ephemeral_id": "abcdef123",
                "name": "elastic-node",
                "version": "3.14.0"
            },
            "client": {
                "ip": "12.53.12.1"
            },
            "container": {
                "id": "container-id"
            },
            "error": {
                "culprit": "my.module.function_name",
                "custom": {
                    "and_objects": {
                        "foo": [
                            "bar",
                            "baz"
                        ]
                    },
                    "my_key": 1,
                    "some_other_value": "foo bar"
                },
                "exception": [
                    {
                        "attributes": {
                            "foo": "bar"
                        },
                        "code": "42",
                        "handled": false,
                        "message": "The username root is unknown",
                        "module": "__builtins__",
                        "stacktrace": [
                            {
                                "abs_path": "/real/file/name.py",
                                "context": {
                                    "post": [
                                        "line4",
                                        "line5"
                                    ],
                                    "pre": [
                                        "line1",
                                        "line2"
                                    ]
                                },
                                "exclude_from_grouping": false,
                                "filename": "file/name.py",
                                "function": "foo",
                                "library_frame": true,
                                "line": {
                                    "column": 4,
                                    "context": "line3",
                                    "number": 3
                                },
                                "module": "App::MyModule",
                                "vars": {
                                    "key": "value"
                                }
                            },
                            {
                                "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js",
                                "context": {
                                    "post": [
                                        "    ins.currentTransaction = prev",
                                        "    return result",
                                        "}",
                                        "}",
                                        "",
                                        "Instrumentation.prototype._recoverTransaction = function (trans) {",
                                        "  if (this.currentTransaction === trans) return"
                                    ],
                                    "pre": [
                                        "  var trans = this.currentTransaction",
                                        "",
                                        "  return instrumented",
                                        "",
                                        "  function instrumented () {",
                                        "    var prev = ins.currentTransaction",
                                        "    ins.currentTransaction = trans"
                                    ]
                                },
                                "exclude_from_grouping": false,
                                "filename": "lib/instrumentation/index.js",
                                "function": "instrumented",
                                "line": {
                                    "context": "    var result = original.apply(this, arguments)",
                                    "number": 102
                                },
                                "vars": {
                                    "key": "value"
                                }
                            }
                        ],
                        "type": "DbError"
                    },
                    {
                        "message": "something wrong writing a file",
                        "type": "InternalDbError"
                    },
                    {
                        "message": "disk spinning way too fast",
                        "type": "VeryInternalDbError"
                    },
                    {
                        "message": "on top of it, internet doesn't work",
                        "parent": 1,
                        "type": "ConnectionError"
                    }
                ],
                "grouping_key": "54c66a6c750b186b2f99d54084934292",
                "id": "0123456789012345",
                "log": {
                    "level": "warning",
                    "logger_name": "my.logger.name",
                    "message": "My service could not talk to the database named foobar",
                    "param_message": "My service could not talk to the database named %s",
                    "stacktrace": [
                        {
                            "abs_path": "/real/file/name.py",
                            "context": {
                                "post": [
                                    "line4",
                                    "line5"
                                ],
                                "pre": [
                                    "line1",
                                    "line2"
                                ]
                            },
                            "exclude_from_grouping": false,
                            "filename": "/webpack/file/name.py",
                            "function": "foo",
                            "library_frame": false,
                            "line": {
                                "column": 4,
                                "context": "line3",
                                "number": 3
                            },
                            "module": "App::MyModule",
                            "vars": {
                                "key": "value"
                            }
                        },
                        {
                            "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js",
                            "context": {
                                "post": [
                                    "    ins.currentTransaction = prev",
                                    "    return result",
                                    "}",
                                    "}",
                                    "",
                                    "Instrumentation.prototype._recoverTransaction = function (trans) {",
                                    "  if (this.currentTransaction === trans) return"
                                ],
                                "pre": [
                                    "  var trans = this.currentTransaction",
                                    "",
                                    "  return instrumented",
                                    "",
                                    "  function instrumented () {",
                                    "    var prev = ins.currentTransaction",
                                    "    ins.currentTransaction = trans"
                                ]
                            },
                            "exclude_from_grouping": false,
                            "filename": "lib/instrumentation/index.js",
                            "function": "instrumented",
                            "line": {
                                "context": "    var result = original.apply(this, arguments)",
                                "number": 102
                            },
                            "vars": {
                                "key": "value"
                            }
                        }
                    ]
                },
                "page": {
                    "referer": "http://localhost:8000/test/e2e/",
                    "url": "http://localhost:8000/test/e2e/general-usecase/"
                }
            },
            "host": {
                "architecture": "x64",
                "hostname": "node-name",
                "ip": "192.0.0.1",
                "name": "prod.example",
                "os": {
                    "platform": "darwin"
                }
            },
            "http": {
                "request": {
                    "body": {
                        "original": "Hello World"
                    },
                    "cookies": {
                        "c1": "v1",
                        "c2": "v2"
                    },
                    "env": {
                        "GATEWAY_INTERFACE": "CGI/1.1",
                        "SERVER_SOFTWARE": "nginx"
                    },
                    "headers": {
                        "Array": [
                            "foo",
                            "bar",
                            "baz"
                        ],
                        "Content-Type": [
                            "text/html"
                        ],
                        "Cookie": [
                            "c1=v1,c2=v2"
                        ],
                        "Some-Other-Header": [
                            "foo"
                        ],
                        "User-Agent": [
                            "Mozilla Chrome Edge"
                        ]
                    },
                    "method": "post",
                    "socket": {
                        "encrypted": true,
                        "remote_address": "12.53.12.1"
                    }
                },
                "response": {
                    "finished": true,
                    "headers": {
                        "Content-Type": [
                            "application/json"
                        ]
                    },
                    "headers_sent": true,
                    "status_code": 200
                },
                "version": "1.1"
            },
            "kubernetes": {
                "namespace": "namespace1",
                "node": {
                    "name": "node-name"
                },
                "pod": {
                    "name": "pod-name",
                    "uid": "pod-uid"
                }
            },
            "labels": {
                "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8"
            },
            "process": {
                "args": [
                    "node",
                    "server.js"
                ],
                "pid": 1234,
                "ppid": 6789,
                "title": "node"
            },
            "processor": {
                "event": "error",
                "name": "error"
            },
            "service": {
                "environment": "staging",
                "framework": {
                    "name": "Node",
                    "version": "1"
                },
                "language": {
                    "name": "ecmascript",
                    "version": "1.2"
                },
                "name": "service1",
                "node": {
                    "name": "node-xyz"
                },
                "runtime": {
                    "name": "node",
                    "version": "8.0.0"
                },
                "version": "5.1.3"
            },
            "source": {
                "ip": "12.53.12.1"
            },
            "timestamp": {
                "us": 1494342245999999
            },
            "url": {
                "domain": "www.example.com",
                "fragment": "#hash",
                "full": "https://www.example.com/p/a/t/h?query=string#hash",
                "original": "/p/a/t/h?query=string#hash",
                "path": "/p/a/t/h",
                "port": 8080,
                "query": "?query=string",
                "scheme": "https"
            },
            "user": {
                "id": "99",
                "name": "foo"
            },
            "user_agent": {
                "original": "Mozilla Chrome Edge"
            }
        },
        {
            "@timestamp": "2018-08-09T14:59:05.999Z",
            "agent": {
                "ephemeral_id": "abcdef123",
                "name": "elastic-node",
                "version": "3.14.0"
            },
            "container": {
                "id": "container-id"
            },
            "error": {
                "exception": [
                    {
                        "message": "Cannot read property 'baz' no defined"
                    }
                ],
                "grouping_key": "ae0232fed4cb40e7ebc62a585a421d60",
                "id": "cdefab0123456789"
            },
            "host": {
                "architecture": "x64",
                "hostname": "node-name",
                "ip": "192.0.0.1",
                "name": "prod.example",
                "os": {
                    "platform": "darwin"
                }
            },
            "kubernetes": {
                "namespace": "namespace1",
                "node": {
                    "name": "node-name"
                },
                "pod": {
                    "name": "pod-name",
                    "uid": "pod-uid"
                }
            },
            "process": {
                "args": [
                    "node",
                    "server.js"
                ],
                "pid": 1234,
                "ppid": 6789,
                "title": "node"
            },
            "processor": {
                "event": "error",
                "name": "error"
            },
            "service": {
                "environment": "staging",
                "framework": {
                    "name": "Express",
                    "version": "1.2.3"
                },
                "language": {
                    "name": "ecmascript",
                    "version": "8"
                },
                "name": "1234_service-12a3",
                "node": {
                    "name": "node-abc"
                },
                "runtime": {
                    "name": "node",
                    "version": "8.0.0"
                },
                "version": "5.1.3"
            },
            "timestamp": {
                "us": 1533826745999000
            },
            "user": {
                "email": "bar@example.com",
                "id": "123",
                "name": "bar"
            }
        },
        {
            "@timestamp": "2018-08-01T10:00:00Z",
            "agent": {
                "ephemeral_id": "justanid",
                "name": "elastic-ruby",
                "version": "2.1.3"
            },
            "container": {
                "id": "container-id"
            },
            "error": {
                "exception": [
                    {
                        "type": "DbError"
                    }
                ],
                "grouping_key": "c3868d6704b923014eaffea034e70a3d",
                "id": "cdefab0123456780"
            },
            "host": {
                "architecture": "x64",
                "hostname": "node-name",
                "ip": "192.0.0.1",
                "name": "prod.example",
                "os": {
                    "platform": "darwin"
                }
            },
            "kubernetes": {
                "namespace": "namespace1",
                "node": {
                    "name": "node-name"
                },
                "pod": {
                    "name": "pod-name",
                    "uid": "pod-uid"
                }
            },
            "parent": {
                "id": "9632587410abcdef"
            },
            "process": {
                "args": [
                    "node",
                    "server.js"
                ],
                "pid": 1234,
                "ppid": 6789,
                "title": "node"
            },
            "processor": {
                "event": "error",
                "name": "error"
            },
            "service": {
                "environment": "testing",
                "framework": {
                    "name": "Rails",
                    "version": "5.0"
                },
                "language": {
                    "name": "ruby",
                    "version": "2.5"
                },
                "name": "service1",
                "node": {
                    "name": "node-abc"
                },
                "runtime": {
                    "name": "cruby",
                    "version": "2.5"
                },
                "version": "2"
            },
            "timestamp": {
                "us": 1533117600000000
            },
            "trace": {
                "id": "0123456789abcdeffedcba0123456789"
            },
            "user": {
                "email": "bar@example.com",
                "id": "123",
                "name": "bar"
            }
        },
        {
            "@timestamp": "2018-08-09T15:04:05.999Z",
            "agent": {
                "ephemeral_id": "abcdef123",
                "name": "elastic-node",
                "version": "3.14.0"
            },
            "container": {
                "id": "container-id"
            },
            "error": {
                "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb",
                "id": "abcdef0123456789",
                "log": {
                    "level": "custom log level",
                    "message": "Cannot read property 'baz' of undefined"
                }
            },
            "host": {
                "architecture": "x64",
                "hostname": "node-name",
                "ip": "192.0.0.1",
                "name": "prod.example",
                "os": {
                    "platform": "darwin"
                }
            },
            "kubernetes": {
                "namespace": "namespace1",
                "node": {
                    "name": "node-name"
                },
                "pod": {
                    "name": "pod-name",
                    "uid": "pod-uid"
                }
            },
            "parent": {
                "id": "9632587410abcdef"
            },
            "process": {
                "args": [
                    "node",
                    "server.js"
                ],
                "pid": 1234,
                "ppid": 6789,
                "title": "node"
            },
            "processor": {
                "event": "error",
                "name": "error"
            },
            "service": {
                "environment": "staging",
                "framework": {
                    "name": "Express",
                    "version": "1.2.3"
                },
                "language": {
                    "name": "ecmascript",
                    "version": "8"
                },
                "name": "1234_service-12a3",
                "node": {
                    "name": "node-abc"
                },
                "runtime": {
                    "name": "node",
                    "version": "8.0.0"
                },
                "version": "5.1.3"
            },
            "timestamp": {
                "us": 1533827045999000
            },
            "trace": {
                "id": "0123456789abcdeffedcba0123456789"
            },
            "transaction": {
                "id": "1234567890987654",
                "sampled": true,
                "type": "request"
            },
            "user": {
                "email": "bar@example.com",
                "id": "123",
                "name": "bar"
            }
        }
    ]
}