No internet access from container in Docker Swarm

I have a problem setting up overlay network in docker swarm (just one manager node). Simply, I cannot connect to internet from within container sitting in docker swarm cluster. When starting container (eg. nginx) on the same host but not in docker swarm cluster I do can connect to internet from within nginx container.

Just to be more precise from within container (in docker swarm):

  • curl google.com works fine
  • ping google.com works fine
  • ping 8.8.8.8 works fine
  • telnet google.com 443 it does not work (but in seperate, no swarm container this works well)

Here's my docker-compose file:


services:
  traefik:
    image: traefik:v2.2
    ports:
      - 443:443
    deploy:
      placement:
        constraints:
          - node.labels.traefik-public.traefik-public-certificates == true
      labels:
        - traefik.enable=true
        - traefik.docker.network=traefik-public
        - traefik.constraint-label=traefik-public
        - traefik.http.middlewares.admin-auth.basicauth.users=admin:${HASHED_PASSWORD?Variable not set}
        - traefik.http.middlewares.https-redirect.redirectscheme.scheme=https
        - traefik.http.middlewares.https-redirect.redirectscheme.permanent=true
        - traefik.http.routers.traefik-public-http.rule=Host(`traefik-beta.example.com`)
        - traefik.http.routers.traefik-public-http.entrypoints=http
        - traefik.http.routers.traefik-public-http.middlewares=https-redirect
        - traefik.http.routers.traefik-public-https.rule=Host(`traefik-beta.example.com`)
        - traefik.http.routers.traefik-public-https.entrypoints=https
        - traefik.http.routers.traefik-public-https.tls=true
        - traefik.http.routers.traefik-public-https.service=api@internal
        - traefik.http.routers.traefik-public-https.tls.certresolver=le
        - traefik.http.routers.traefik-public-https.middlewares=admin-auth
        - traefik.http.services.traefik-public.loadbalancer.server.port=8080
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - traefik-public-certificates:/certificates
    command:
      - --providers.docker
      - --providers.docker.constraints=Label(`traefik.constraint-label`, `traefik-public`)
      - --providers.docker.exposedbydefault=false
      - --providers.docker.swarmmode
      - --providers.docker.endpoint=unix:///var/run/docker.sock
      - --entrypoints.http.address=:80
      - --entrypoints.https.address=:443
      - --certificatesresolvers.le.acme.email=accounts@example.com
      - --certificatesresolvers.le.acme.storage=/certificates/acme.json
      - --certificatesresolvers.le.acme.tlschallenge=true
      - --accesslog
      - --log
      - --api
    networks:
      - traefik-public
  nextjs:
    image: $WEB_IMAGE
    deploy:
      replicas: 1
      update_config:
        parallelism: 1
        order: start-first
        failure_action: rollback
        delay: 10s
      rollback_config:
        parallelism: 0
        order: stop-first
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 10
        window: 120s
      labels:
        - traefik.enable=true
        - traefik.docker.network=traefik-public
        - traefik.constraint-label=traefik-public
        - traefik.http.middlewares.https-redirect.redirectscheme.scheme=https
        - traefik.http.middlewares.https-redirect.redirectscheme.permanent=true
        - traefik.http.routers.web-public-http.rule=Host(`web-beta.example.com`)
        - traefik.http.routers.web-public-http.entrypoints=http
        - traefik.http.routers.web-public-http.middlewares=https-redirect
        - traefik.http.routers.web-public-https.rule=Host(`web-beta.example.com`)
        - traefik.http.routers.web-public-https.entrypoints=https
        - traefik.http.routers.web-public-https.tls=true
        - traefik.http.routers.web-public-https.tls.certresolver=le
        - traefik.http.services.nextjs.loadbalancer.server.port=3000
    ports:
      - 3000:3000
    healthcheck:
      test: "${DOCKER_HEALTHCHECK_TEST:-curl localhost:3000}"
    networks:
      - traefik-public
    environment:
      API_HOST: http://server:3000
      BASE_HOST: https://web-beta.example.com
      NODE_ENV: production
  server:
    image: $API_IMAGE
    deploy:
      replicas: 1
      update_config:
        parallelism: 1
        order: start-first
        failure_action: rollback
        delay: 10s
      rollback_config:
        parallelism: 0
        order: stop-first
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 12
        window: 120s
      labels:
        - traefik.enable=true
        - traefik.docker.network=traefik-public
        - traefik.constraint-label=traefik-public
        - traefik.http.middlewares.https-redirect.redirectscheme.scheme=https
        - traefik.http.middlewares.https-redirect.redirectscheme.permanent=true
        - traefik.http.routers.api-public-http.rule=Host(`api-beta.example.com`)
        - traefik.http.routers.api-public-http.entrypoints=http
        - traefik.http.routers.api-public-http.middlewares=https-redirect
        - traefik.http.routers.api-public-https.rule=Host(`api-beta.example.com`)
        - traefik.http.routers.api-public-https.entrypoints=https
        - traefik.http.routers.api-public-https.tls=true
        - traefik.http.routers.api-public-https.tls.certresolver=le
        - traefik.http.services.api.loadbalancer.server.port=3000
    environment:
      DATABASE_URL: postgres://postgres:$PG_PASSWORD@db:5432/db_production
      REDIS_URL: redis://cache:6379/1
      RAILS_ENV: production
      FRONTEND_APP_URL: http://nextjs
    volumes:
      - '.:/project'
    healthcheck:
      test: "${DOCKER_HEALTHCHECK_TEST:-curl localhost:3000}"
    networks:
      - traefik-public
  db:
    image: postgres:11
    volumes:
      - db_data:/var/lib/postgresql/data
    deploy:
      replicas: 1
      update_config:
        parallelism: 1
        order: start-first
        failure_action: rollback
        delay: 10s
      rollback_config:
        parallelism: 0
        order: stop-first
      restart_policy:
        condition: any
        delay: 5s
        max_attempts: 10
        window: 300s
    networks:
      - traefik-public
    environment:
      POSTGRES_USERNAME: 'postgres'
      POSTGRES_PASSWORD: $PG_PASSWORD
      POSTGRES_DB: 'db_production'

volumes:
  traefik-public-certificates:
  db_data:

networks:
  traefik-public:
    external: true
    attachable: true

When inspecting traefik-public overlay network it is:

[
    {
        "Name": "traefik-public",
        "Id": "58z41wiog0gmzibjd3gaaf0wv",
        "Created": "2021-06-10T09:06:40.649003923Z",
        "Scope": "swarm",
        "Driver": "overlay",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "193.161.1.0/24",
                    "Gateway": "193.161.1.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": true,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": // here is all containers configuration object
        "Options": {
            "com.docker.network.driver.overlay.vxlanid_list": "4097"
        },
        "Labels": {},
        "Peers": [
            {
                "Name": "52264af1c1f826",
                "IP": "xxx.xxx.xx.xx"
            }
        ]
    }
]

Can you give any hints how to resolve it?