How to proxy WebSockets in Traefik?

Traefik handle websocket, and you don't need any specific configuration for this.

Your problem seems to be more about the challenge in Let's Encrypt. Let's Encrypt doesn't handle TLS Challenge on other port than the default one and the default challenging in Traefik is TLS :(

So you need to configure Traefik to use DNS Challenge https://docs.traefik.io/configuration/acme/

Worked example for confluence

version: '3.3'

networks:
  traefik:
    external: true

volumes:
  portainer_data:
  confluence:

services:
  traefik:
    image: traefik:1.7.9-alpine
    command: >
      --docker
      --docker.swarmmode
      --docker.watch
      --docker.exposedbydefault=true
      --docker.domain=example.com
      --defaultentrypoints=http,https,ws,wss
      --entrypoints='Name:http Address::80'
      --entrypoints='Name:https Address::443 TLS'
      --acme
      --acme.email='[email protected]'
      --acme.storage='/certs/acme.json'
      --acme.entryPoint=https
      --acme.httpChallenge.entryPoint=http
      --acme.onhostrule=true
      --acme.acmelogging=true
      --logLevel=INFO
      --accessLog
      --api
    ports:
      - 80:80
      - 443:443
    networks:
      - manager
      - traefik
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /certs:/certs
    deploy:
      mode: global
      labels:
        - traefik.enable=true
        - traefik.port=8080
        - traefik.frontend.rule=Host:traefik.example.com
        - traefik.docker.network=traefik
        #- traefik.redirectorservice.frontend.entryPoints=http
        #- traefik.redirectorservice.frontend.redirect.entryPoint=https
        - traefik.webservice.frontend.entryPoints=http,https
  portainer:
    image: portainer/portainer:1.20.1
    command: -H tcp://tasks.agent:9001 --tlsskipverify
    networks:
      - manager
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - portainer_data:/data
    deploy:
      placement:
        constraints: [node.role == manager]
      labels:
        - traefik.enable=true
        - traefik.port=9000
        - traefik.frontend.rule=Host:portainer.example.com
        - traefik.docker.network=traefik
        #- traefik.redirectorservice.frontend.entryPoints=http
        #- traefik.redirectorservice.frontend.redirect.entryPoint=https
        - traefik.webservice.frontend.entryPoints=http,https
  agent:
    image: portainer/agent:1.2.1
    environment:
      AGENT_CLUSTER_ADDR: tasks.agent
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - manager
    deploy:
      mode: global
  confluence:
    image: cptactionhank/atlassian-confluence:6.3.4
    networks:
      - traefik
    volumes:
      - confluence:/var/atlassian/confluence
    deploy:
      replicas: 1
      labels:
        - traefik.enable=true
        - traefik.port=8090
        - traefik.frontend.rule=Host:confluence.example.com
        - traefik.docker.network=traefik
        # - traefik.redirectorservice.frontend.entryPoints=http
        # - traefik.redirectorservice.frontend.redirect.entryPoint=https
        - traefik.webservice.frontend.entryPoints=http,https,ws,wss