Instrument Erlang/Elixir applications with OpenTelemetry

Add the following dependencies to your rebar.config file:

erlangcopydownload
{deps, [
  {opentelemetry_api, "~> 1.1.1"},
  {opentelemetry, "~> 1.1.2"},
  {opentelemetry_exporter, "~> 1.2.2"}
]}.

Add the following dependencies to your src/<PROJ_NAME>.app.src file:

erlangcopydownload
{applications, [kernel,
                stdlib,
                opentelemetry_api,
                opentelemetry,
                opentelemetry_exporter]}

Create a file named extra_metadata.erl within your src/ directory, containing the following:

erlangcopydownload
-module(extra_metadata).
-behaviour(otel_resource_detector).
-export([get_resource/1]).

get_resource(_) ->
    Metadata = otel_resource:create(otel_resource_app_env:parse(get_metadata("/var/lib/dynatrace/enrichment/dt_metadata.properties")), []),
    {ok, MetadataFilePath} = file:read_file("dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties"),
    Metadata2 = otel_resource:create(otel_resource_app_env:parse(get_metadata(MetadataFilePath)), []),
    otel_resource:merge(Metadata, Metadata2).

get_metadata(FileName) ->
try
    {ok, MetadataFile} = file:read_file(FileName),
    Lines = binary:split(MetadataFile, <<"\n">>, [trim, global]),
    make_tuples(Lines, [])
catch _:_ -> "Metadata not found, safe to continue"
end.

make_tuples([Line|Lines], Acc) ->
    [Key, Value] = binary:split(Line, <<"=">>),
    make_tuples(Lines, [{Key, Value}|Acc]);
make_tuples([], Acc) -> Acc.

Set names for the span and add attributes as you see fit.

erlangcopydownload
-export([init/2]).

-include_lib("opentelemetry_api/include/otel_tracer.hrl").
-include_lib("opentelemetry/include/otel_resource.hrl").

init( Req, State ) ->
    ?with_span(<<"parent_span">>, #{attributes => [ %%TODO Add span name
      {<<"my-key-1">>, <<"my-value-1">>}] %%TODO Add attributes at span creation
    }, fun child_function/1),
    %% Your code goes here


child_function(_SpanCtx) ->
    ?with_span(<<"child_span">>, #{},
               fun(_ChildSpanCtx) ->
                ?set_attributes([{<<"child-key-1">>, <<"child-value-1">>}]) %%TODO Add attributes after span creation
               end).

Add the following dependencies to your mix.exs file:

elixircopydownload
defp deps do
  [
    {:opentelemetry_api, "~> 1.1"},
    {:opentelemetry, "~> 1.1"},
    {:opentelemetry_exporter, "~> 1.2"}
  ]
end

Also add a releases section to your mix.exs file:

elixircopydownload
def application do
  [
    extra_applications: [:logger],
    mod: {<PROJ_NAME>.Application, []},
    releases: [
      <project_name>: [
        version: "<project_version>",
        applications: [opentelemetry_exporter: :permanent, opentelemetry: :temporary]
      ]
    ]
  ]
end

Create a file named extra_metadata.ex within your lib directory, with the following:

elixircopydownload
defmodule ExtraMetadata do

  @behaviour :otel_resource_detector

  def get_resource(_) do
    metadata = read_file("/var/lib/dynatrace/enrichment/dt_metadata.properties") |> unwrap_lines
    file_path = read_file("dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties") |> unwrap_lines
    metadata2 = read_file(file_path) |> unwrap_lines
    attributes = get_attributes(Enum.concat(metadata, metadata2))
    :otel_resource.create(attributes)
  end

  defp unwrap_lines({:ok, metadata}), do: metadata
  defp unwrap_lines({:error, _}), do: []

  defp read_file(file_name) do
    try do
      {:ok, String.split(File.read!(file_name), "\n")}
    rescue
      File.Error ->
        {:error, "File does not exist, safe to continue"}
    end
  end

  defp get_attributes(metadata) do
    Enum.map(metadata, fn(line) ->
      if String.length(line) > 0 do
        [key, value] = String.split(line, "=")
        {key, value}
      else
        {:error, "EOF"}
      end
    end)
  end
end

Set names for the span and add attributes as you see fit.

elixircopydownload
require OpenTelemetry.Tracer, as: Tracer

def hello do
  Tracer.with_span "my-span", %{attributes: [{<<"my-key-1">>, <<"my-value-1">>}]} do #TODO add attributes at span creation
    Tracer.set_attributes([{"another-key-1", "another-value-1"}]) #TODO add attributes after span creation
    # Your code goes here
  end
end

To send data to Dynatrace with OneAgent, add the following configuration to your config/sys.config file:

erlangcopydownload
[
  {otel_getting_started, []},

  {opentelemetry,
    [{span_processor, batch},
      {traces_exporter, otlp},
      {resource,
        [{service,
          #{name => "erlang-quickstart", version => "1.0.1"} %%TODO Replace with the name and version of your application
        }]
      },
      {resource_detectors, [
        otel_resource_env_var,
        otel_resource_app_env,
        extra_metadata
      ]}
    ]
  },
  {opentelemetry_exporter,
    [{otlp_protocol, http_protobuf},
      {otlp_traces_endpoint, "http://localhost:14499/otlp/v1/traces"}
    ]}
].

When using OneAgent, make sure to enable the public Extension Execution Controller in your Dynatrace Settings, otherwise no data will be sent.

In the Dynatrace menu, go to Settings > Preferences > Extension Execution Controller. The toggles Enable Extension Execution Controller and Enable local PIPE/HTTP metric and Log Ingest API should be active.

To send data to Dynatrace without OneAgent, add the following configuration to your config/sys.config file:

erlangcopydownload
[
  {otel_getting_started, []},

  {opentelemetry,
    [{span_processor, batch},
      {traces_exporter, otlp},
      {resource,
        [{service,
          #{name => "erlang-quickstart", version => "1.0.1"} %%TODO Replace with the name and version of your application
        }]
      },
      {resource_detectors, [
        otel_resource_env_var,
        otel_resource_app_env,
        extra_metadata
      ]}
    ]
  },
  {opentelemetry_exporter,
    [{otlp_protocol, http_protobuf},
      {otlp_traces_endpoint, "<URL>"}, %%TODO Replace <URL> to your SaaS/Managed URL as mentioned in the next step
      {otlp_headers, [{"Authorization", "Api-Token <TOKEN>"}]} %%TODO Replace <TOKEN> with your API Token as mentioned in the next step
    ]}
].

Lastly, you need to define the correct endpoint and token, to make sure your data arrives where it should be.

• To set the endpoint:
  1. Use your Environment ID to set the endpoint to which your app will send traces as follows:
     • Dynatrace SaaS https://{your-environment-id}.live.dynatrace.com/api/v2/otlp/v1/traces
     • Dynatrace Managed https://{your-domain}/e/{your-environment-id}/api/v2/otlp/v1/traces
     • Dynatrace ActiveGate https://{your-activegate-endpoint}/e/{your-environment-id}/api/v2/otlp/v1/traces
       • You may need to include the port to your ActiveGate endpoint. For example:
         • https://{your-activegate-endpoint}:9999/e/{your-environment-id}/api/v2/otlp/v1/traces
       • If you are running a containerized ActiveGate, you need to use the FQDN of it. For example:
         • https://{your-activegate-service-name}.dynatrace.svc.cluster.local/e/{your-environment-id}/api/v2/otlp/v1/traces
  2. Replace <URL> in the code snippet above with your endpoint.
• To create an authentication token
  1. In the Dynatrace menu, go to Access tokens and select Generate new token.
  2. Provide a Token name.
  3. In the Search scopes box, search for Ingest OpenTelemetry traces and select the checkbox.
  4. Select Generate token.
  5. Select Copy to copy the token to your clipboard.
  6. Save the token in a safe place; you can't display it again.
  7. Replace <TOKEN> in the code snippet above with your token.

To send data to Dynatrace with OneAgent, edit your config/config.exs file (or create the file if it does not already exist) and add the following configuration:

elixircopydownload
import Config

config :opentelemetry,
  resource: [service: %{name: "elixir-quickstart", version: "1.0.1"}], #TODO Replace with the name and version of your application
  span_processor: :batch,
  traces_exporter: :otlp,
  resource_detectors: [
    :otel_resource_app_env,
    :otel_resource_env_var,
    ExtraMetadata
  ]

config :opentelemetry_exporter,
  otlp_protocol: :http_protobuf,
  otlp_traces_endpoint: "http://localhost:14499/otlp/v1/traces"

When using OneAgent, make sure to enable the public Extension Execution Controller in your Dynatrace Settings, otherwise no data will be sent.

In the Dynatrace menu, go to Settings > Preferences > Extension Execution Controller. The toggles Enable Extension Execution Controller and Enable local PIPE/HTTP metric and Log Ingest API should be active.

To send data to Dynatrace without OneAgent, edit your config/runtime.exs file (or create the file if it does not already exist) and add the following configuration:

elixircopydownload
import Config

config :opentelemetry,
  resource: [service: %{name: "elixir-quickstart", version: "1.0.1"}], #TODO Replace with the name and version of your application
  span_processor: :batch,
  traces_exporter: :otlp,
  resource_detectors: [
    :otel_resource_app_env,
    :otel_resource_env_var,
    ExtraMetadata
  ]

config :opentelemetry_exporter,
  otlp_protocol: :http_protobuf,
  otlp_traces_endpoint: "<URL>", #TODO Replace <URL> to your SaaS/Managed URL as mentioned in the next step
  otlp_traces_headers: [{"Authorization", "Api-Token <TOKEN>"}] #TODO Replace <TOKEN> with your API Token as mentioned in the next step

Lastly, you need to define the correct endpoint and token, to make sure your data arrives where it should be.

• To set the endpoint:
  1. Use your Environment ID to set the endpoint to which your app will send traces as follows:
     • Dynatrace SaaS https://{your-environment-id}.live.dynatrace.com/api/v2/otlp/v1/traces
     • Dynatrace Managed https://{your-domain}/e/{your-environment-id}/api/v2/otlp/v1/traces
     • Dynatrace ActiveGate https://{your-activegate-endpoint}/e/{your-environment-id}/api/v2/otlp/v1/traces
       • You may need to include the port to your ActiveGate endpoint. For example:
         • https://{your-activegate-endpoint}:9999/e/{your-environment-id}/api/v2/otlp/v1/traces
       • If you are running a containerized ActiveGate, you need to use the FQDN of it. For example:
         • https://{your-activegate-service-name}.dynatrace.svc.cluster.local/e/{your-environment-id}/api/v2/otlp/v1/traces
  2. Replace <URL> in the code snippet above with your endpoint.
• To create an authentication token
  1. In the Dynatrace menu, go to Access tokens and select Generate new token.
  2. Provide a Token name.
  3. In the Search scopes box, search for Ingest OpenTelemetry traces and select the checkbox.
  4. Select Generate token.
  5. Select Copy to copy the token to your clipboard.
  6. Save the token in a safe place; you can't display it again.
  7. Replace <TOKEN> in the code snippet above with your token.

Add the following sys.config file:

erlangcopydownload
...
{text_map_propagators, [baggage,
                        trace_context]},
...

The snippets below show examples where otel_propagator_text_map is used.

erlangcopydownload
?with_span(<<"span-name">>, #{},
    fun(_ChildSpanCtx) ->

        %% a custom header example
        Headers = [{"content-type", "application/json"}, {"X-Custom-Header", "some-value"}],

        %% we convert the traceparent information and merge the 2 headers as
        %% httpc:request requires tuples of strings
        Tmp = [],
        NewHeaders = headers_list(otel_propagator_text_map:inject(opentelemetry:get_text_map_injector(), Tmp)),
        MergedHeaders = lists:append(Headers, NewHeaders),

        {ok, Res} = httpc:request(get, {URL, MergedHeaders}, [], []),
        io:format("Response: ~p~n", [Res])
    end).

headers_list(Headers) ->
    [{binary_to_list(Name), binary_to_list(Value)} || {Name, Value} <- Headers].

erlangcopydownload
%% Get Headers from incoming request and propagate the Context
Headers = maps:get(headers, Req),
otel_propagator_text_map:extract(maps:to_list(Headers)),

SpanCtx = ?start_span(<<"span-name">>),

%% As we used `otel_propagator_text_map` the current context is from the parent span
Ctx = otel_ctx:get_current(),

proc_lib:spawn_link(fun() ->
    %% Start span and set as current
    otel_ctx:attach(Ctx),
    ?set_current_span(SpanCtx),

    %% Create response
    Resp = cowboy_req:reply(
        200,
        #{<<"content-type">> => <<"application/json">>},
        <<"{\"message\": \"hello world\"}">>,
        Req
    ),

    {ok, Resp, State},
    ?end_span(SpanCtx)

Add the following to your runtime.exs file:

elixircopydownload
...
text_map_propagators: [:baggage, :trace_context],
...

The snippets below show examples where otel_propagator_text_map is used.

elixircopydownload
OpenTelemetry.Tracer.with_span "span-name" do
    ...
    # do work here
    ...
    headers = [{"content-type", "application/json"}, {"X-Custom-Header", "some-value"}]

    merged_headers = :otel_propagator_text_map.inject(headers)

    case HTTPoison.get(URL, merged_headers, []) do
        {:ok, res} -> IO.puts("Response: #{inspect(res)}")
        {:error, _} -> raise "request failed"

    end
end

elixircopydownload
#extract headers
:otel_propagator_text_map.extract(conn.req_headers)

span_ctx = OpenTelemetry.Tracer.start_span(<<"span-name">>)
ctx = OpenTelemetry.Ctx.get_current()

task = Task.async(fn ->
    OpenTelemetry.Ctx.attach(ctx)
    OpenTelemetry.Tracer.set_current_span(span_ctx)
    # do work here

    OpenTelemetry.Tracer.end_span(span_ctx)
end)