How to configure Jaeger with elasticsearch?

If you would like to deploy the Jaeger with Elasticsearch and Kibana to quickly validate and check the stack e.g. in kind or Minikube, the following snippet may help you.

#######################
## Add jaegertracing helm repo
#######################
helm repo add jaegertracing
https://jaegertracing.github.io/helm-charts

#######################
## Create a target namespace
#######################
kubectl create namespace observability

#######################
## Check and use the jaegertracing helm chart
#######################
helm search repo jaegertracing
helm install -n observability jaeger-operator jaegertracing/jaeger-operator

#######################
## Use the elasticsearch all-in-one operator
#######################
kubectl apply -f https://download.elastic.co/downloads/eck/1.1.2/all-in-one.yaml

#######################
## Create an elasticsearch deployment
#######################
cat <<EOF | kubectl apply -n observability -f -
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: quickstart
spec:
  version: 7.7.0
  nodeSets:
  - name: default
    count: 1
    config:
      node.master: true
      node.data: true
      node.ingest: true
      node.store.allow_mmap: false
EOF

PASSWORD=$(kubectl get secret -n observability quickstart-es-elastic-user -o=jsonpath='{.data.elastic}' | base64 --decode)
kubectl create secret -n observability generic jaeger-secret --from-literal=ES_PASSWORD=${PASSWORD} --from-literal=ES_USERNAME=elastic

#######################
## Kibana to visualize the trace data
#######################

cat <<EOF | kubectl apply -n observability -f -
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
  name: quickstart
spec:
  version: 7.7.0
  count: 1
  elasticsearchRef:
    name: quickstart
EOF

kubectl port-forward -n observability service/quickstart-kb-http 5601
## To get the pw
kubectl get secret quickstart-es-elastic-user -o=jsonpath='{.data.elastic}' | base64 --decode; echo

login:
https://localhost:5601
username: elastic
pw: <see above to outcome of the command>

#######################
## Deploy a jaeger tracing application
#######################
cat <<EOF | kubectl apply -n observability -f -
apiVersion: jaegertracing.io/v1
kind: Jaeger
metadata:
  name: simple-prod
spec:
  agent:
    strategy: DaemonSet
  strategy: production
  storage:
    type: elasticsearch
    options:
      es:
        server-urls: https://quickstart-es-http:9200
        tls:
          ca: /es/certificates/ca.crt
        num-shards: 1
        num-replicas: 0
    secretName: jaeger-secret
  volumeMounts:
    - name: certificates
      mountPath: /es/certificates/
      readOnly: true
  volumes:
    - name: certificates
      secret:
        secretName: quickstart-es-http-certs-public
EOF

## to visualize it
kubectl --namespace observability port-forward simple-prod-query-<POP ID> 16686:16686

#######################
## To test the setup
## Of course if you set it up to another namespace it will work, the only thing that matters is the collector URL and PORT
#######################
cat <<EOF | kubectl apply -n observability -f -
apiVersion: v1
kind: List
items:
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    name: jaeger-k8s-example
    labels:
      app: jaeger-k8s-example
  spec:
    replicas: 1
    selector:
      matchLabels:
        app: jaeger-k8s-example
    strategy:
      type: Recreate
    template:
      metadata:
        labels:
          app: jaeger-k8s-example
      spec:
        containers:
          - name: jaeger-k8s-example
            env:
            - name: JAEGER_COLLECTOR_URL
              value: "simple-prod-collector.observability.svc.cluster.local"
            - name: JAEGER_COLLECTOR_PORT
              value: "14268"
            image: norbertfenk/jaeger-k8s-example:latest
            imagePullPolicy: IfNotPresent
EOF

As I mentioned in my comment on the OP's first answer above, I was getting an error when running the docker-compose exactly as given:

Error: unknown flag: --collector.host-port

I think this CLI flag has been deprecated by the Jaeger folks since that answer was written. So I poked around in the jaeger-agent documentation a bit:

  1. https://www.jaegertracing.io/docs/1.20/deployment/#discovery-system-integration
  2. https://www.jaegertracing.io/docs/1.20/cli/#jaeger-agent

And I got this to work with a couple of small modifications:

  1. I added port range "14250:14250" to the jaeger-collector ports
  2. I updated the jaeger-agent command input with: command: ["--reporter.grpc.host-port=jaeger-collector:14250"]
  3. Finally, I updated the ellastic search version in the image tag to the latest version they have available at this time (though I doubt this was required).

The updated docker-compose.yaml:

version: "3"

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
    networks:
      - elastic-jaeger
    ports:
      - "127.0.0.1:9200:9200"
      - "127.0.0.1:9300:9300"
    restart: on-failure
    environment:
      - cluster.name=jaeger-cluster
      - discovery.type=single-node
      - http.host=0.0.0.0
      - transport.host=127.0.0.1
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=false
    volumes:
      - esdata:/usr/share/elasticsearch/data

  jaeger-collector:
    image: jaegertracing/jaeger-collector
    ports:
      - "14269:14269"
      - "14268:14268"
      - "14267:14267"
      - "14250:14250"
      - "9411:9411"
    networks:
      - elastic-jaeger
    restart: on-failure
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
    command: [
      "--es.server-urls=http://elasticsearch:9200",
      "--es.num-shards=1",
      "--es.num-replicas=0",
      "--log-level=error"
    ]
    depends_on:
      - elasticsearch

  jaeger-agent:
    image: jaegertracing/jaeger-agent
    hostname: jaeger-agent
    command: ["--reporter.grpc.host-port=jaeger-collector:14250"]
    ports:
      - "5775:5775/udp"
      - "6831:6831/udp"
      - "6832:6832/udp"
      - "5778:5778"
    networks:
      - elastic-jaeger
    restart: on-failure
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
    depends_on:
      - jaeger-collector

  jaeger-query:
    image: jaegertracing/jaeger-query
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - no_proxy=localhost
    ports:
      - "16686:16686"
      - "16687:16687"
    networks:
      - elastic-jaeger
    restart: on-failure
    command: [
      "--es.server-urls=http://elasticsearch:9200",
      "--span-storage.type=elasticsearch",
      "--log-level=debug"
    ]
    depends_on:
      - jaeger-agent

volumes:
  esdata:
    driver: local

networks:
  elastic-jaeger:
    driver: bridge

After searching a solution for some time, I found a docker-compose.yml file which had the Jaeger Query,Agent,collector and Elasticsearch configurations.

docker-compose.yml

  version: "3"

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
    networks:
      - elastic-jaeger
    ports:
      - "127.0.0.1:9200:9200"
      - "127.0.0.1:9300:9300"
    restart: on-failure
    environment:
      - cluster.name=jaeger-cluster
      - discovery.type=single-node
      - http.host=0.0.0.0
      - transport.host=127.0.0.1
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=false
    volumes:
      - esdata:/usr/share/elasticsearch/data

  jaeger-collector:
    image: jaegertracing/jaeger-collector
    ports:
      - "14269:14269"
      - "14268:14268"
      - "14267:14267"
      - "9411:9411"
    networks:
      - elastic-jaeger
    restart: on-failure
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
    command: [
      "--es.server-urls=http://elasticsearch:9200",
      "--es.num-shards=1",
      "--es.num-replicas=0",
      "--log-level=error"
    ]
    depends_on:
      - elasticsearch

  jaeger-agent:
    image: jaegertracing/jaeger-agent
    hostname: jaeger-agent
    command: ["--collector.host-port=jaeger-collector:14267"]
    ports:
      - "5775:5775/udp"
      - "6831:6831/udp"
      - "6832:6832/udp"
      - "5778:5778"
    networks:
      - elastic-jaeger
    restart: on-failure
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
    depends_on:
      - jaeger-collector

  jaeger-query:
    image: jaegertracing/jaeger-query
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - no_proxy=localhost
    ports:
      - "16686:16686"
      - "16687:16687"
    networks:
      - elastic-jaeger
    restart: on-failure
    command: [
      "--es.server-urls=http://elasticsearch:9200",
      "--span-storage.type=elasticsearch",
      "--log-level=debug"
    ]
    depends_on:
      - jaeger-agent

volumes:
  esdata:
    driver: local

networks:
  elastic-jaeger:
    driver: bridge

The docker-compose.yml file installs the elasticsearch, Jaeger collector,query and agent.

Install docker and docker compose first https://docs.docker.com/compose/install/#install-compose

Then, execute these commands in order

 
1. sudo docker-compose up -d elasticsearch

2. sudo docker-compose up -d 

3. sudo docker ps -a

start all the docker containers - Jaeger agent,collector,query and elasticsearch.

sudo docker start container-id

access -> http://localhost:16686/

Tags:

Docker

Elasticsearch

Opentracing

Jaeger

Distributed Tracing

Recent Posts