From cf7c791e0ebebbbaf2d6468731063246fd4b4e55 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 22 Aug 2022 21:39:53 +0100 Subject: [PATCH] feat: Workers (#1) * add workers and mem limits * disable health checks on fed workers * add example worker configs * update workers to make them consistent * add docs for worker config --- README.md | 4 + docker-compose.yaml | 99 ++++++++++++++- example-worker-configs/README.md | 119 ++++++++++++++++++ .../federation_sender.log.config | 29 +++++ .../synapse-federation-sender-1.yaml | 14 +++ .../synapse-federation-sender-2.yaml | 14 +++ .../synapse-generic-worker-1.yaml | 19 +++ .../synapse-generic-worker-2.yaml | 19 +++ example-worker-configs/worker.log.config | 29 +++++ 9 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 example-worker-configs/README.md create mode 100644 example-worker-configs/federation_sender.log.config create mode 100644 example-worker-configs/synapse-federation-sender-1.yaml create mode 100644 example-worker-configs/synapse-federation-sender-2.yaml create mode 100644 example-worker-configs/synapse-generic-worker-1.yaml create mode 100644 example-worker-configs/synapse-generic-worker-2.yaml create mode 100644 example-worker-configs/worker.log.config diff --git a/README.md b/README.md index 08544de..6b60b1b 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,10 @@ database: 5. Locate the `public_baseurl` setting in `homeserver.yaml` and set it to the URL of your Matrix server domain name (e.g. `matrix.example.com`) +### Configure Synapse Workers + +The provided [docker-compose.yaml](docker-compose.yaml) contains 4 workers (2 generic and 2 federation senders). See [Synapse Workers docs](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) for the different types. See the [Example worker configs README](example-worker-configs/README.md) for more detail. + 6. Bring up the containers! ```bash diff --git a/docker-compose.yaml b/docker-compose.yaml index 23c2287..afb0f2e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -38,8 +38,9 @@ services: logging: driver: "json-file" options: - max-size: "1G" + max-size: "100M" max-file: "3" + mem_limit: ${SYNAPSE_MEM_LIMIT:-2g} ports: - ${SYNAPSE_PORT}:8008 networks: @@ -53,3 +54,99 @@ services: TZ: ${TZ} depends_on: - postgres + + synapse-generic-worker-1: + image: matrixdotorg/synapse:${SYNAPSE_IMAGE_TAG} + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/${CONFIG_FILE_NAME}", "--config-path=/data/workers/synapse-generic-worker-1.yaml"] + healthcheck: + test: ["CMD-SHELL", "curl -fSs http://localhost:8081/health || exit 1"] + start_period: "5s" + interval: "15s" + timeout: "5s" + volumes: + - ${VOLUME_PATH}/data:/data:rw + logging: + driver: "json-file" + options: + max-size: "100M" + max-file: "3" + mem_limit: ${WORKER_MEM_LIMIT:-512m} + networks: + - frontend + - backend + environment: + SYNAPSE_CONFIG_DIR: /data + SYNAPSE_WORKER: synapse.app.generic_worker + depends_on: + - synapse + + synapse-generic-worker-2: + image: matrixdotorg/synapse:${SYNAPSE_IMAGE_TAG} + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/${CONFIG_FILE_NAME}", "--config-path=/data/workers/synapse-generic-worker-2.yaml"] + healthcheck: + test: ["CMD-SHELL", "curl -fSs http://localhost:8081/health || exit 1"] + start_period: "5s" + interval: "15s" + timeout: "5s" + volumes: + - ${VOLUME_PATH}/data:/data:rw + logging: + driver: "json-file" + options: + max-size: "100M" + max-file: "3" + mem_limit: ${WORKER_MEM_LIMIT:-512m} + networks: + - frontend + - backend + environment: + SYNAPSE_CONFIG_DIR: /data + SYNAPSE_WORKER: synapse.app.generic_worker + depends_on: + - synapse + + synapse-federation-sender-1: + image: matrixdotorg/synapse:${SYNAPSE_IMAGE_TAG} + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/${CONFIG_FILE_NAME}", "--config-path=/data/workers/synapse-federation-sender-1.yaml"] + healthcheck: + disable: true + volumes: + - ${VOLUME_PATH}/data:/data:rw + logging: + driver: "json-file" + options: + max-size: "100M" + max-file: "3" + mem_limit: ${WORKER_MEM_LIMIT:-512m} + networks: + - backend + environment: + SYNAPSE_CONFIG_DIR: /data + SYNAPSE_WORKER: synapse.app.federation_sender + depends_on: + - synapse + + synapse-federation-sender-2: + image: matrixdotorg/synapse:${SYNAPSE_IMAGE_TAG} + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/${CONFIG_FILE_NAME}", "--config-path=/data/workers/synapse-federation-sender-2.yaml"] + healthcheck: + disable: true + volumes: + - ${VOLUME_PATH}/data:/data:rw + logging: + driver: "json-file" + options: + max-size: "100M" + max-file: "3" + mem_limit: ${WORKER_MEM_LIMIT:-512m} + networks: + - backend + environment: + SYNAPSE_CONFIG_DIR: /data + SYNAPSE_WORKER: synapse.app.federation_sender + depends_on: + - synapse diff --git a/example-worker-configs/README.md b/example-worker-configs/README.md new file mode 100644 index 0000000..b7cbba0 --- /dev/null +++ b/example-worker-configs/README.md @@ -0,0 +1,119 @@ +# Setting up Synapse with Workers using Docker Compose + +Example worker configuration files can be found here. + +## Worker Service Examples in Docker Compose + +In order to start the Synapse container as a worker, you must specify an `entrypoint` that loads both the `homeserver.yaml` and the configuration for the worker (`synapse-generic-worker-1.yaml` in the example below). You must also include the worker type in the environment variable `SYNAPSE_WORKER` or alternatively pass `-m synapse.app.generic_worker` as part of the `entrypoint` after `"/start.py", "run"`). + +### Generic Worker Example + +```yaml +synapse-generic-worker-1: + image: matrixdotorg/synapse:latest + container_name: synapse-generic-worker-1 + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-generic-worker-1.yaml"] + healthcheck: + test: ["CMD-SHELL", "curl -fSs http://localhost:8081/health || exit 1"] + start_period: "5s" + interval: "15s" + timeout: "5s" + volumes: + - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume + environment: + SYNAPSE_WORKER: synapse.app.generic_worker + # Expose port if required so your reverse proxy can send requests to this worker + # Port configuration will depend on how the http listener is defined in the worker configuration file + ports: + - 8081:8081 + depends_on: + - synapse +``` + +### Federation Sender Example + +Please note: The federation sender does not receive REST API calls so no exposed ports are required. + +```yaml +synapse-federation-sender-1: + image: matrixdotorg/synapse:latest + container_name: synapse-federation-sender-1 + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-federation-sender-1.yaml"] + healthcheck: + disable: true + volumes: + - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume + environment: + SYNAPSE_WORKER: synapse.app.federation_sender + depends_on: + - synapse +``` + +## `homeserver.yaml` Configuration + +### Enable Redis + +Locate the `redis` section of your `homeserver.yaml` and enable and configure it: + +```yaml +redis: + enabled: true + host: redis + port: 6379 + # password: +``` + +This assumes that your Redis service is called `redis` in your Docker Compose file. + +### Add a replication Listener + +Locate the `listeners` section of your `homeserver.yaml` and add the following replication listener: + +```yaml +listeners: + # Other listeners + + - port: 9093 + type: http + resources: + - names: [replication] +``` + +This listener is used by the workers for replication and is referred to in worker config files using the following settings: + +```yaml +worker_replication_host: synapse +worker_replication_http_port: 9093 +``` + +### Add Workers to `instance_map` + +Locate the `instance_map` section of your `homeserver.yaml` and populate it with your workers: + +```yaml +instance_map: + synapse-generic-worker-1: # The worker_name setting in your worker configuration file + host: synapse-generic-worker-1 # The name of the worker service in your Docker Compose file + port: 8034 # The port assigned to the replication listener in your worker config file + synapse-federation-sender-1: + host: synapse-federation-sender-1 + port: 8034 +``` + +### Configure Federation Senders + +This section is applicable if you are using Federation senders (synapse.app.federation_sender). Locate the `send_federation` and `federation_sender_instances` settings in your `homeserver.yaml` and configure them: + +```yaml +# This will disable federation sending on the main Synapse instance +send_federation: false + +federation_sender_instances: + - synapse-federation-sender-1 # The worker_name setting in your federation sender worker configuration file +``` + +## Other Worker types + +Using the concepts shown here it is possible to create other worker types in Docker Compose. See the [Workers](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) documentation for a list of available workers. \ No newline at end of file diff --git a/example-worker-configs/federation_sender.log.config b/example-worker-configs/federation_sender.log.config new file mode 100644 index 0000000..0e93c7a --- /dev/null +++ b/example-worker-configs/federation_sender.log.config @@ -0,0 +1,29 @@ +version: 1 + +formatters: + precise: + + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + + +handlers: + + + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO + +root: + level: WARN + + + handlers: [console] + + +disable_existing_loggers: false diff --git a/example-worker-configs/synapse-federation-sender-1.yaml b/example-worker-configs/synapse-federation-sender-1.yaml new file mode 100644 index 0000000..32a06d1 --- /dev/null +++ b/example-worker-configs/synapse-federation-sender-1.yaml @@ -0,0 +1,14 @@ +worker_app: synapse.app.federation_sender +worker_name: synapse-federation-sender-1 + +# The replication listener on the main synapse process. +worker_replication_host: synapse +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + +worker_log_config: /data/workers/federation_sender.log.config diff --git a/example-worker-configs/synapse-federation-sender-2.yaml b/example-worker-configs/synapse-federation-sender-2.yaml new file mode 100644 index 0000000..22e766f --- /dev/null +++ b/example-worker-configs/synapse-federation-sender-2.yaml @@ -0,0 +1,14 @@ +worker_app: synapse.app.federation_sender +worker_name: synapse-federation-sender-2 + +# The replication listener on the main synapse process. +worker_replication_host: synapse +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + +worker_log_config: /data/workers/federation_sender.log.config diff --git a/example-worker-configs/synapse-generic-worker-1.yaml b/example-worker-configs/synapse-generic-worker-1.yaml new file mode 100644 index 0000000..5816b53 --- /dev/null +++ b/example-worker-configs/synapse-generic-worker-1.yaml @@ -0,0 +1,19 @@ +worker_app: synapse.app.generic_worker +worker_name: synapse-generic-worker-1 + +# The replication listener on the main synapse process. +worker_replication_host: synapse +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + - type: http + port: 8081 + x_forwarded: true + resources: + - names: [client, federation] + +worker_log_config: /data/workers/worker.log.config diff --git a/example-worker-configs/synapse-generic-worker-2.yaml b/example-worker-configs/synapse-generic-worker-2.yaml new file mode 100644 index 0000000..70273e7 --- /dev/null +++ b/example-worker-configs/synapse-generic-worker-2.yaml @@ -0,0 +1,19 @@ +worker_app: synapse.app.generic_worker +worker_name: synapse-generic-worker-2 + +# The replication listener on the main synapse process. +worker_replication_host: synapse +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + - type: http + port: 8081 + x_forwarded: true + resources: + - names: [client, federation] + +worker_log_config: /data/workers/worker.log.config diff --git a/example-worker-configs/worker.log.config b/example-worker-configs/worker.log.config new file mode 100644 index 0000000..0e93c7a --- /dev/null +++ b/example-worker-configs/worker.log.config @@ -0,0 +1,29 @@ +version: 1 + +formatters: + precise: + + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + + +handlers: + + + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO + +root: + level: WARN + + + handlers: [console] + + +disable_existing_loggers: false