Instrument Erlang/Elixir applications with OpenTelemetry
This guide shows how to instrument your Erlang/Elixir application with OpenTelemetry and export the traces to Dynatrace.
To learn more about how Dynatrace works with OpenTelemetry, see Send data to Dynatrace with OpenTelemetry.
Prerequisites
- Dynatrace version 1.222+
- W3C Trace Context is enabled
- From the Dynatrace menu, go to Settings > Preferences > OneAgent features.
- Turn on Send W3C Trace Context HTTP headers.
Overview
To monitor your Erlang/Elixir application with OpenTelemetry
Instrument your application
Send the data to Dynatrace
Configure context propagation
Restart your application and verify the data in Dynatrace
Configure data capturing to meet privacy requirements
Instrument your application
To instrument manually, add the dependencies and the code snippet below to any Erlang or Elixir method you want to monitor.
Add the following dependencies to your rebar.config
file:
{deps, [
{opentelemetry_api, "~> 1.1.1"},
{opentelemetry, "~> 1.1.2"},
{opentelemetry_exporter, "~> 1.2.2"}
]}.
Add the following dependencies to your src/<PROJ_NAME>.app.src
file:
{applications, [kernel,
stdlib,
opentelemetry_api,
opentelemetry,
opentelemetry_exporter]}
Create a file named extra_metadata.erl
within your src/
directory, containing the following:
-module(extra_metadata).
-behaviour(otel_resource_detector).
-export([get_resource/1]).
get_resource(_) ->
Metadata = otel_resource:create(otel_resource_app_env:parse(get_metadata("/var/lib/dynatrace/enrichment/dt_metadata.properties")), []),
{ok, MetadataFilePath} = file:read_file("dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties"),
Metadata2 = otel_resource:create(otel_resource_app_env:parse(get_metadata(MetadataFilePath)), []),
otel_resource:merge(Metadata, Metadata2).
get_metadata(FileName) ->
try
{ok, MetadataFile} = file:read_file(FileName),
Lines = binary:split(MetadataFile, <<"\n">>, [trim, global]),
make_tuples(Lines, [])
catch _:_ -> "Metadata not found, safe to continue"
end.
make_tuples([Line|Lines], Acc) ->
[Key, Value] = binary:split(Line, <<"=">>),
make_tuples(Lines, [{Key, Value}|Acc]);
make_tuples([], Acc) -> Acc.
Set names for the span and add attributes as you see fit.
-export([init/2]).
-include_lib("opentelemetry_api/include/otel_tracer.hrl").
-include_lib("opentelemetry/include/otel_resource.hrl").
init( Req, State ) ->
?with_span(<<"parent_span">>, #{attributes => [ %%TODO Add span name
{<<"my-key-1">>, <<"my-value-1">>}] %%TODO Add attributes at span creation
}, fun child_function/1),
%% Your code goes here
child_function(_SpanCtx) ->
?with_span(<<"child_span">>, #{},
fun(_ChildSpanCtx) ->
?set_attributes([{<<"child-key-1">>, <<"child-value-1">>}]) %%TODO Add attributes after span creation
end).
Add the following dependencies to your mix.exs
file:
defp deps do
[
{:opentelemetry_api, "~> 1.1"},
{:opentelemetry, "~> 1.1"},
{:opentelemetry_exporter, "~> 1.2"}
]
end
Also add a releases
section to your mix.exs
file:
def application do
[
extra_applications: [:logger],
mod: {<PROJ_NAME>.Application, []},
releases: [
<project_name>: [
version: "<project_version>",
applications: [opentelemetry_exporter: :permanent, opentelemetry: :temporary]
]
]
]
end
Create a file named extra_metadata.ex
within your lib
directory, with the following:
defmodule ExtraMetadata do
@behaviour :otel_resource_detector
def get_resource(_) do
metadata = read_file("/var/lib/dynatrace/enrichment/dt_metadata.properties") |> unwrap_lines
file_path = read_file("dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties") |> unwrap_lines
metadata2 = read_file(file_path) |> unwrap_lines
attributes = get_attributes(Enum.concat(metadata, metadata2))
:otel_resource.create(attributes)
end
defp unwrap_lines({:ok, metadata}), do: metadata
defp unwrap_lines({:error, _}), do: []
defp read_file(file_name) do
try do
{:ok, String.split(File.read!(file_name), "\n")}
rescue
File.Error ->
{:error, "File does not exist, safe to continue"}
end
end
defp get_attributes(metadata) do
Enum.map(metadata, fn(line) ->
if String.length(line) > 0 do
[key, value] = String.split(line, "=")
{key, value}
else
{:error, "EOF"}
end
end)
end
end
Set names for the span and add attributes as you see fit.
require OpenTelemetry.Tracer, as: Tracer
def hello do
Tracer.with_span "my-span", %{attributes: [{<<"my-key-1">>, <<"my-value-1">>}]} do #TODO add attributes at span creation
Tracer.set_attributes([{"another-key-1", "another-value-1"}]) #TODO add attributes after span creation
# Your code goes here
end
end
Send data to Dynatrace
To send data to Dynatrace, adjust the config files as below:
To send data to Dynatrace with OneAgent, add the following configuration to your config/sys.config
file:
[
{otel_getting_started, []},
{opentelemetry,
[{span_processor, batch},
{traces_exporter, otlp},
{resource,
[{service,
#{name => "erlang-quickstart", version => "1.0.1"} %%TODO Replace with the name and version of your application
}]
},
{resource_detectors, [
otel_resource_env_var,
otel_resource_app_env,
extra_metadata
]}
]
},
{opentelemetry_exporter,
[{otlp_protocol, http_protobuf},
{otlp_traces_endpoint, "http://localhost:14499/otlp/v1/traces"}
]}
].
When using OneAgent, make sure to enable the public Extension Execution Controller in your Dynatrace Settings, otherwise no data will be sent.
In the Dynatrace menu, go to Settings > Preferences > Extension Execution Controller. The toggles Enable Extension Execution Controller and Enable local PIPE/HTTP metric and Log Ingest API should be active.
To send data to Dynatrace without OneAgent, add the following configuration to your config/sys.config
file:
[
{otel_getting_started, []},
{opentelemetry,
[{span_processor, batch},
{traces_exporter, otlp},
{resource,
[{service,
#{name => "erlang-quickstart", version => "1.0.1"} %%TODO Replace with the name and version of your application
}]
},
{resource_detectors, [
otel_resource_env_var,
otel_resource_app_env,
extra_metadata
]}
]
},
{opentelemetry_exporter,
[{otlp_protocol, http_protobuf},
{otlp_traces_endpoint, "<URL>"}, %%TODO Replace <URL> to your SaaS/Managed URL as mentioned in the next step
{otlp_headers, [{"Authorization", "Api-Token <TOKEN>"}]} %%TODO Replace <TOKEN> with your API Token as mentioned in the next step
]}
].
Lastly, you need to define the correct endpoint and token, to make sure your data arrives where it should be.
- To set the endpoint:
- Use your Environment ID to set the endpoint to which your app will send traces as follows:
- Dynatrace SaaS
https://{your-environment-id}.live.dynatrace.com/api/v2/otlp/v1/traces
- Dynatrace Managed
https://{your-domain}/e/{your-environment-id}/api/v2/otlp/v1/traces
- Dynatrace ActiveGate
https://{your-activegate-endpoint}/e/{your-environment-id}/api/v2/otlp/v1/traces
- You may need to include the port to your ActiveGate endpoint. For example:
https://{your-activegate-endpoint}:9999/e/{your-environment-id}/api/v2/otlp/v1/traces
- If you are running a containerized ActiveGate, you need to use the FQDN of it. For example:
https://{your-activegate-service-name}.dynatrace.svc.cluster.local/e/{your-environment-id}/api/v2/otlp/v1/traces
- You may need to include the port to your ActiveGate endpoint. For example:
- Dynatrace SaaS
- Replace
<URL>
in the code snippet above with your endpoint.
- Use your Environment ID to set the endpoint to which your app will send traces as follows:
- To create an authentication token
- In the Dynatrace menu, go to Access tokens and select Generate new token.
- Provide a Token name.
- In the Search scopes box, search for
Ingest OpenTelemetry traces
and select the checkbox. - Select Generate token.
- Select Copy to copy the token to your clipboard.
- Save the token in a safe place; you can't display it again.
- Replace
<TOKEN>
in the code snippet above with your token.
To send data to Dynatrace with OneAgent, edit your config/config.exs
file (or create the file if it does not already exist) and add the following configuration:
import Config
config :opentelemetry,
resource: [service: %{name: "elixir-quickstart", version: "1.0.1"}], #TODO Replace with the name and version of your application
span_processor: :batch,
traces_exporter: :otlp,
resource_detectors: [
:otel_resource_app_env,
:otel_resource_env_var,
ExtraMetadata
]
config :opentelemetry_exporter,
otlp_protocol: :http_protobuf,
otlp_traces_endpoint: "http://localhost:14499/otlp/v1/traces"
When using OneAgent, make sure to enable the public Extension Execution Controller in your Dynatrace Settings, otherwise no data will be sent.
In the Dynatrace menu, go to Settings > Preferences > Extension Execution Controller. The toggles Enable Extension Execution Controller and Enable local PIPE/HTTP metric and Log Ingest API should be active.
To send data to Dynatrace without OneAgent, edit your config/runtime.exs
file (or create the file if it does not already exist) and add the following configuration:
import Config
config :opentelemetry,
resource: [service: %{name: "elixir-quickstart", version: "1.0.1"}], #TODO Replace with the name and version of your application
span_processor: :batch,
traces_exporter: :otlp,
resource_detectors: [
:otel_resource_app_env,
:otel_resource_env_var,
ExtraMetadata
]
config :opentelemetry_exporter,
otlp_protocol: :http_protobuf,
otlp_traces_endpoint: "<URL>", #TODO Replace <URL> to your SaaS/Managed URL as mentioned in the next step
otlp_traces_headers: [{"Authorization", "Api-Token <TOKEN>"}] #TODO Replace <TOKEN> with your API Token as mentioned in the next step
Lastly, you need to define the correct endpoint and token, to make sure your data arrives where it should be.
- To set the endpoint:
- Use your Environment ID to set the endpoint to which your app will send traces as follows:
- Dynatrace SaaS
https://{your-environment-id}.live.dynatrace.com/api/v2/otlp/v1/traces
- Dynatrace Managed
https://{your-domain}/e/{your-environment-id}/api/v2/otlp/v1/traces
- Dynatrace ActiveGate
https://{your-activegate-endpoint}/e/{your-environment-id}/api/v2/otlp/v1/traces
- You may need to include the port to your ActiveGate endpoint. For example:
https://{your-activegate-endpoint}:9999/e/{your-environment-id}/api/v2/otlp/v1/traces
- If you are running a containerized ActiveGate, you need to use the FQDN of it. For example:
https://{your-activegate-service-name}.dynatrace.svc.cluster.local/e/{your-environment-id}/api/v2/otlp/v1/traces
- You may need to include the port to your ActiveGate endpoint. For example:
- Dynatrace SaaS
- Replace
<URL>
in the code snippet above with your endpoint.
- Use your Environment ID to set the endpoint to which your app will send traces as follows:
- To create an authentication token
- In the Dynatrace menu, go to Access tokens and select Generate new token.
- Provide a Token name.
- In the Search scopes box, search for
Ingest OpenTelemetry traces
and select the checkbox. - Select Generate token.
- Select Copy to copy the token to your clipboard.
- Save the token in a safe place; you can't display it again.
- Replace
<TOKEN>
in the code snippet above with your token.
Configure context propagation optional
To propagate trace context between services, you need to configure your OpenTelemetry application. Global TextMap Propagators can be configured in your application environment.
Add the following sys.config
file:
...
{text_map_propagators, [baggage,
trace_context]},
...
The snippets below show examples where otel_propagator_text_map
is used.
?with_span(<<"span-name">>, #{},
fun(_ChildSpanCtx) ->
%% a custom header example
Headers = [{"content-type", "application/json"}, {"X-Custom-Header", "some-value"}],
%% we convert the traceparent information and merge the 2 headers as
%% httpc:request requires tuples of strings
Tmp = [],
NewHeaders = headers_list(otel_propagator_text_map:inject(opentelemetry:get_text_map_injector(), Tmp)),
MergedHeaders = lists:append(Headers, NewHeaders),
{ok, Res} = httpc:request(get, {URL, MergedHeaders}, [], []),
io:format("Response: ~p~n", [Res])
end).
headers_list(Headers) ->
[{binary_to_list(Name), binary_to_list(Value)} || {Name, Value} <- Headers].
%% Get Headers from incoming request and propagate the Context
Headers = maps:get(headers, Req),
otel_propagator_text_map:extract(maps:to_list(Headers)),
SpanCtx = ?start_span(<<"span-name">>),
%% As we used `otel_propagator_text_map` the current context is from the parent span
Ctx = otel_ctx:get_current(),
proc_lib:spawn_link(fun() ->
%% Start span and set as current
otel_ctx:attach(Ctx),
?set_current_span(SpanCtx),
%% Create response
Resp = cowboy_req:reply(
200,
#{<<"content-type">> => <<"application/json">>},
<<"{\"message\": \"hello world\"}">>,
Req
),
{ok, Resp, State},
?end_span(SpanCtx)
Add the following to your runtime.exs
file:
...
text_map_propagators: [:baggage, :trace_context],
...
The snippets below show examples where otel_propagator_text_map
is used.
OpenTelemetry.Tracer.with_span "span-name" do
...
# do work here
...
headers = [{"content-type", "application/json"}, {"X-Custom-Header", "some-value"}]
merged_headers = :otel_propagator_text_map.inject(headers)
case HTTPoison.get(URL, merged_headers, []) do
{:ok, res} -> IO.puts("Response: #{inspect(res)}")
{:error, _} -> raise "request failed"
end
end
#extract headers
:otel_propagator_text_map.extract(conn.req_headers)
span_ctx = OpenTelemetry.Tracer.start_span(<<"span-name">>)
ctx = OpenTelemetry.Ctx.get_current()
task = Task.async(fn ->
OpenTelemetry.Ctx.attach(ctx)
OpenTelemetry.Tracer.set_current_span(span_ctx)
# do work here
OpenTelemetry.Tracer.end_span(span_ctx)
end)
Verify that the traces are ingested into Dynatrace
A few minutes after restarting your app, look for your spans:
- In the Dynatrace menu, go to Distributed traces and select the Ingested traces tab.
- If you use OneAgent, go to Distributed traces and select the PurePaths tab.
- Your spans will be part of an existing PurePath, if the root of your call is already monitored by OneAgent.
If your application does not receive any traffic, there will be no traces.
Configure data capture to meet privacy requirements optional
While Dynatrace automatically captures all OpenTelemetry resource and span attributes, only attribute values specified in the allowlist are stored and displayed in the Dynatrace web UI. This prevents accidental storage of personal data, so you can meet your privacy requirements and control the amount of monitoring data that's stored.
To view your custom span attributes, you need to allow them in the Dynatrace web UI first:
- Span attributes In the Dynatrace menu, go to Settings and select Server-side service monitoring > Span attributes.
- Resource attributes In the Dynatrace menu, go to Settings and select Server-side service monitoring > Resource attributes.