From 5d21be20d57f86604ec06af601b47a05f9318e76 Mon Sep 17 00:00:00 2001
From: Pierre Couy <>
Date: Wed, 4 Dec 2024 17:05:13 +0100
Subject: [PATCH] Disable resource-hungry clickhouse logging & metrics

Also includes a bunch of smaller optimizations
---
 .gitignore                            |  3 ++-
 clickhouse/clickhouse-config.xml      | 17 +++++++++++++
 clickhouse/clickhouse-user-config.xml | 17 +++++++++++++
 clickhouse/logs.xml                   | 36 ---------------------------
 compose.yml                           |  5 ++--
 5 files changed, 38 insertions(+), 40 deletions(-)
 create mode 100644 clickhouse/clickhouse-config.xml
 create mode 100644 clickhouse/clickhouse-user-config.xml
 delete mode 100644 clickhouse/logs.xml

diff --git a/.gitignore b/.gitignore
index 4e33a3c..329cc50 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 *
 !compose.yml
-!clickhouse/logs.xml
+!clickhouse/clickhouse-config.xml
+!clickhouse/clickhouse-user-config.xml
 !clickhouse/ipv4-only.xml
 !README.md
 !LICENSE
diff --git a/clickhouse/clickhouse-config.xml b/clickhouse/clickhouse-config.xml
new file mode 100644
index 0000000..886c9e6
--- /dev/null
+++ b/clickhouse/clickhouse-config.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <logger>
+        <level>warning</level>
+        <console>true</console>
+    </logger>
+
+    <!-- Stop all the unnecessary logging -->
+    <query_thread_log remove="remove"/>
+    <query_metric_log remove="remove"/>
+    <query_log remove="remove"/>
+    <text_log remove="remove"/>
+    <trace_log remove="remove"/>
+    <metric_log remove="remove"/>
+    <asynchronous_metric_log remove="remove"/>
+    <session_log remove="remove"/>
+    <part_log remove="remove"/>
+</clickhouse>
diff --git a/clickhouse/clickhouse-user-config.xml b/clickhouse/clickhouse-user-config.xml
new file mode 100644
index 0000000..d128cc7
--- /dev/null
+++ b/clickhouse/clickhouse-user-config.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <log_queries>0</log_queries>
+            <log_query_threads>0</log_query_threads>
+            <max_block_size>1024</max_block_size>
+            <max_download_threads>1</max_download_threads>
+            <input_format_parallel_parsing>0</input_format_parallel_parsing>
+            <output_format_parallel_formatting>0</output_format_parallel_formatting>
+        </default>
+    </profiles>
+
+    <asynchronous_metrics_update_period_s>60</asynchronous_metrics_update_period_s>
+    <dns_cache_update_period>90</dns_cache_update_period>
+    <mark_cache_size>524288000</mark_cache_size> 
+
+</clickhouse>
diff --git a/clickhouse/logs.xml b/clickhouse/logs.xml
deleted file mode 100644
index 0eac674..0000000
--- a/clickhouse/logs.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<clickhouse>
-    <logger>
-        <level>warning</level>
-        <console>true</console>
-    </logger>
-
-    <query_log replace="1">
-        <database>system</database>
-        <table>query_log</table>
-        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
-        <engine>
-            ENGINE = MergeTree
-            PARTITION BY event_date
-            ORDER BY (event_time)
-            TTL event_date + interval 30 day
-            SETTINGS ttl_only_drop_parts=1
-        </engine>
-    </query_log>
-
-    <metric_log>
-        <partition_by>event_date</partition_by>
-        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
-    </metric_log>
-
-    <asynchronous_metric_log>
-        <partition_by>event_date</partition_by>
-        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
-    </asynchronous_metric_log>
-
-    <!-- Stops unnecessary logging -->
-    <query_thread_log remove="remove" />
-    <text_log remove="remove" />
-    <trace_log remove="remove" />
-    <session_log remove="remove" />
-    <part_log remove="remove" />
-</clickhouse>
diff --git a/compose.yml b/compose.yml
index ec0af50..527dad4 100644
--- a/compose.yml
+++ b/compose.yml
@@ -14,9 +14,8 @@ services:
     image: clickhouse/clickhouse-server:24.11.1.2557-alpine
     restart: always
     volumes:
-      - event-data:/var/lib/clickhouse
-      - event-logs:/var/log/clickhouse-server
-      - ./clickhouse/logs.xml:/etc/clickhouse-server/config.d/logs.xml:ro
+      - ./clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro
+      - ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/config.d/user.xml:ro
       # This makes ClickHouse bind to IPv4 only, since Docker doesn't enable IPv6 in bridge networks by default.
       # Fixes "Listen [::]:9000 failed: Address family for hostname not supported" warnings.
       - ./clickhouse/ipv4-only.xml:/etc/clickhouse-server/config.d/ipv4-only.xml:ro