Connecting log data to traces

{
  "severity": "error",
  "time": 1638957438023,
  "pid": 1,
  "hostname": "paymentservice-788946fdcd-42lgq",
  "name": "paymentservice-charge",
  "dt.trace_id": "d04b42bc9f4b6ecdbf6bc9f4b6ecdbc",
  "dt.span_id": "9adc716eb808d428",
  "dt.entity.process_group_instance": "PROCESS_GROUP_INSTANCE-27204EFED3D8466E",
  "message": "Unsupported card type for cardNumber=************0454"
}

<?xml version="1.0" encoding="windows-1252" standalone="no"?>
<record>
    <date>2021-08-24T14:41:36.565218700Z</date>
    <millis>1629816096565</millis>
    <nanos>218700</nanos>
    <sequence>0</sequence>
    <logger>com.apm.testapp.logging.jul.XMLLoggingSample</logger>
    <level>INFO</level>
    <class>com.apm.testapp.logging.jul.BaseLoggingSample</class>
    <method>info</method>
    <thread>1</thread>
    <message>Update completed successfully.</message>
    <dt.trace_id>513fcd4e9b08792fcd4e9b08792</dt.trace_id>
    <dt.span_id>125840e3125840e3</dt.span_id>
    <dt.entity.process_group_instance>PROCESS_GROUP_INSTANCE-27204EFED3D8466E</dt.entity.process_group_instance>
</record>

127.0.0.1 - [21/Oct/2021:10:33:28 +0200] GET /index.htm HTTP/1.1 404 597 [!dt dt.trace_id=aa764ee37ebaa764ee37eaa764ee37e,dt.span_id=b93ede8b93ede8, dt.entity.process_group_instance=PROCESS_GROUP_INSTANCE-27204EFED3D8466E]

log_format custom '$remote_addr - [$time_local] $request $status $body_bytes_sent [!dt dt.trace_id=$dt_trace_id,dt.span_id=$dt_span_id,dt.trace_sampled=$dt_trace_sampled]';
access_log logs/access.log custom;

127.0.0.1 - [22/Mar/2022:08:50:45 +0100] GET /index.htm HTTP/1.1 200 30 [!dt dt.trace_id=b9e5c9ec08be5fab5071d76f427be7da,dt.span_id=43c5bb9432593963,dt.trace_sampled=true]
127.0.0.1 - [22/Mar/2022:08:50:45 +0100] GET /index.htm HTTP/1.1 200 30 [!dt dt.trace_id=01e52950b145d97bf22345e68c5e6c58,dt.span_id=de819d856eecb236,dt.trace_sampled=true]

log_format custom '$remote_addr - [$time_local] $request $status $body_bytes_sent [!dt dt.trace_id=$trace_id,dt.span_id=$span_id]'; access_log logs/access.log custom

127.0.0.1 - [21/Oct/2021:10:33:28 +0200] GET /index.htm HTTP/1.1 404 597 [!dt dt.trace_id=e1c0afeb0b8a91d7748139aa764ee37e,dt.span_id=e5e6748fab93ede8]
127.0.0.1 - [21/Oct/2021:10:33:31 +0200] GET /index.html HTTP/1.1 200 1056 [!dt dt.trace_id=81fe7816ba6c38f7aa09aef3684cd941,dt.span_id=3bdacc466ae073cd]

<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} dt.trace_id=%X{dt.trace_id} dt.span_id=%X{dt.span_id} dt.entity.process_group_instance=%X{dt.entity.process_group_instance} - %msg%n"/>

<appender name="COMPOSITEJSONENCODER" class="ch.qos.logback.core.FileAppender">
  <file>compositejsonencoder.log</file>
  <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
    <providers>
      <timestamp>
        <fieldName>timestamp</fieldName>
        <timeZone>UTC</timeZone>
      </timestamp>
      <loggerName>
        <fieldName>logger</fieldName>
      </loggerName>
      <logLevel>
        <fieldName>level</fieldName>
      </logLevel>
      <threadName>
        <fieldName>thread</fieldName>
      </threadName>
      <mdc>
        <includeMdcKeyName>dt.span_id</includeMdcKeyName>
        <includeMdcKeyName>dt.trace_id</includeMdcKeyName>
        <includeMdcKeyName>dt.entity.host</includeMdcKeyName>
      </mdc>
      <stackTrace>
        <fieldName>stackTrace</fieldName>
        <!-- maxLength - limit the length of the stack trace -->
        <throwableConverter class="net.logstash.logback.stacktrace.ShortenedThrowableConverter">
          <maxDepthPerThrowable>200</maxDepthPerThrowable>
          <maxLength>14000</maxLength>
          <rootCauseFirst>true</rootCauseFirst>
        </throwableConverter>
      </stackTrace>
      <message />
      <throwableClassName>
        <fieldName>exceptionClass</fieldName>
      </throwableClassName>
    </providers>
  </encoder>
</appender>

main-snippet: load_module /opt/dynatrace/oneagent/agent/bin/current/linux-musl-x86-64/liboneagentnginx.so;
log-format-upstream: '$remote_addr - $remote_user [$time_local] "$request" [!dt dt.trace_id=$dt_trace_id,dt.span_id=$dt_span_id,dt.trace_sampled=$dt_trace_sampled] $status $body_bytes_sent  "$http_referer" "$http_user_agent" $request_length'

import json

from opentelemetry import trace as OpenTelemetry
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
    OTLPSpanExporter,
)
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider, sampling
from opentelemetry.sdk.trace.export import (
    BatchSpanProcessor,
)

merged = dict()
for name in ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.json", "/var/lib/dynatrace/enrichment/dt_metadata.json"]:
    try:
        data = ''
        with open(name) as f:
          data = json.load(f if name.startswith("/var") else open(f.read()))
        merged.update(data)
    except:
        pass

merged.update({
    "service.name": "python-quickstart", #TODO Replace with the name of your application
    "service.version": "1.0.1", #TODO Replace with the version of your application
})
resource = Resource.create(merged)

tracer_provider = TracerProvider(sampler=sampling.ALWAYS_ON, resource=resource)
OpenTelemetry.set_tracer_provider(tracer_provider)

tracer_provider.add_span_processor(
    BatchSpanProcessor(OTLPSpanExporter(
        endpoint="http://localhost:14499/otlp/v1/traces"
    )))

const winston = require("winston");
const Transport = require("winston-transport");

class CustomTransport extends Transport {
  log(info, next) {
    let myLogLine = `MyLogLine: ${info.timestamp} level=${info.level}: ${info.message}`;
    // this is important as above line only picks timestamp, level and message but nothing else from metadata
    if (info["dt.trace_id"]) {
      myLogLine = `[!dt dt.trace_id=${info["dt.trace_id"]},dt.span_id=${info["dt.span_id"]},dt.trace_sampled=${info["dt.trace_sampled"]}] ${myLogLine}`;
    }
    console.log(myLogLine);
    next();
  }
}

const logger = winston.createLogger({
  level: "info",
  format: winston.format.timestamp(),
  transports: [
    new CustomTransport(),
    // this transport includes all metadata (including dynatrace added traceId,..)
    new winston.transport.Console({
      format: winston.format.simple()
    })
  ]
})