Add stop_grace_period of 60s to all containers

This reverts commit 43e8842846.

.github/ISSUE_TEMPLATE/best-place-for-self-hosting-issues-is-the-community-forum.md

@@ -0,0 +1,10 @@
+---
+name: Best place for self-hosting issues is the community forum
+about: Please do not submit issues here
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Please do not submit issues here. Best place for self-hosting issues, questions and ideas is [the community forum here on GitHub](https://github.com/plausible/analytics/discussions/categories/self-hosted-support). Thanks!

.github/workflows/spellcheck.yml

@@ -0,0 +1,20 @@
+name: spellcheck
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  codespell:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: codespell-project/actions-codespell@v2
+        with:
+          check_filenames: true
+
+  typos:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: crate-ci/typos@master

.gitignore

@@ -1,8 +1,2 @@
-*
-!compose.yml
-!clickhouse/logs.xml
-!clickhouse/ipv4-only.xml
-!clickhouse/low-resources.xml
-!README.md
-!LICENSE
-!.gitignore
+data/
+.env

.prettierignore

@@ -0,0 +1 @@
+*

clickhouse/clickhouse-config.xml

@@ -0,0 +1,23 @@
+<clickhouse>
+    <logger>
+        <level>warning</level>
+        <console>true</console>
+    </logger>
+    
+    <!-- 
+    Avoid the warning: "Listen [::]:9009 failed: Address family for hostname not supported". 
+    If Docker has IPv6 disabled, bind ClickHouse to IPv4 to prevent this issue. 
+    Add this to the configuration to ensure it listens on all IPv4 interfaces:
+    <listen_host>0.0.0.0</listen_host>
+    -->
+
+    <!-- Stop all the unnecessary logging -->
+    <query_thread_log remove="remove"/>
+    <query_log remove="remove"/>
+    <text_log remove="remove"/>
+    <trace_log remove="remove"/>
+    <metric_log remove="remove"/>
+    <asynchronous_metric_log remove="remove"/>
+    <session_log remove="remove"/>
+    <part_log remove="remove"/>
+</clickhouse>

clickhouse/clickhouse-user-config.xml

@@ -0,0 +1,8 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <log_queries>0</log_queries>
+            <log_query_threads>0</log_query_threads>
+        </default>
+    </profiles>
+</clickhouse>

clickhouse/ipv4-only.xml

@@ -1,3 +0,0 @@
-<clickhouse>
-    <listen_host>0.0.0.0</listen_host>
-</clickhouse>

clickhouse/logs.xml

@@ -1,28 +0,0 @@
-<clickhouse>
-    <logger>
-        <level>warning</level>
-        <console>true</console>
-    </logger>
-
-    <query_log replace="1">
-        <database>system</database>
-        <table>query_log</table>
-        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
-        <engine>
-            ENGINE = MergeTree
-            PARTITION BY event_date
-            ORDER BY (event_time)
-            TTL event_date + interval 30 day
-            SETTINGS ttl_only_drop_parts=1
-        </engine>
-    </query_log>
-
-    <!-- Stops unnecessary logging -->
-    <metric_log remove="remove" />
-    <asynchronous_metric_log remove="remove" />
-    <query_thread_log remove="remove" />
-    <text_log remove="remove" />
-    <trace_log remove="remove" />
-    <session_log remove="remove" />
-    <part_log remove="remove" />
-</clickhouse>

clickhouse/low-resources.xml

@@ -1,23 +0,0 @@
-<!-- https://clickhouse.com/docs/en/operations/tips#using-less-than-16gb-of-ram -->
-<clickhouse>
-    <!--
-    https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#mark_cache_size -->
-    <mark_cache_size>524288000</mark_cache_size>
-
-    <profile>
-        <default>
-            <!-- https://clickhouse.com/docs/en/operations/settings/settings#max_threads -->
-            <max_threads>1</max_threads>
-            <!-- https://clickhouse.com/docs/en/operations/settings/settings#max_block_size -->
-            <max_block_size>8192</max_block_size>
-            <!-- https://clickhouse.com/docs/en/operations/settings/settings#max_download_threads -->
-            <max_download_threads>1</max_download_threads>
-            <!--
-            https://clickhouse.com/docs/en/operations/settings/settings#input_format_parallel_parsing -->
-            <input_format_parallel_parsing>0</input_format_parallel_parsing>
-            <!--
-            https://clickhouse.com/docs/en/operations/settings/settings#output_format_parallel_formatting -->
-            <output_format_parallel_formatting>0</output_format_parallel_formatting>
-        </default>
-    </profile>
-</clickhouse>

compose.override.yml

@@ -1,4 +0,0 @@
-services:
-  plausible:
-    ports:
-      - 127.0.0.1:84:80

compose.yml

@@ -1,5 +1,7 @@
 services:
   plausible_db:
+    # Plausible v2.1.1 was tested against PostgreSQL versions 15 and 16
+    # https://github.com/plausible/analytics/blob/v2.1.1/.github/workflows/elixir.yml#L21-L32
     image: postgres:16-alpine
     stop_grace_period: 60s
     restart: always
@@ -7,9 +9,6 @@ services:
       - ./data/postgres:/var/lib/postgresql/data
     environment:
       - POSTGRES_PASSWORD=postgres
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
-      start_period: 1m
 
   plausible_events_db:
     image: clickhouse/clickhouse-server:24.3.3.102-alpine
@@ -18,73 +17,22 @@ services:
     volumes:
       - ./data/clickhouse:/var/lib/clickhouse
       - ./data/clickhouse-logs:/var/log/clickhouse-server
-      - ./clickhouse/logs.xml:/etc/clickhouse-server/config.d/logs.xml:ro
-      # This makes ClickHouse bind to IPv4 only, since Docker doesn't enable IPv6 in bridge networks by default.
-      # Fixes "Listen [::]:9000 failed: Address family for hostname not supported" warnings.
-      - ./clickhouse/ipv4-only.xml:/etc/clickhouse-server/config.d/ipv4-only.xml:ro
-      # This makes ClickHouse consume less resources, which is useful for small setups.
-      # https://clickhouse.com/docs/en/operations/tips#using-less-than-16gb-of-ram
-      - ./clickhouse/low-resources.xml:/etc/clickhouse-server/config.d/low-resources.xml:ro
+      - ./clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro
+      - ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/logging.xml:ro
     ulimits:
       nofile:
         soft: 262144
         hard: 262144
-    healthcheck:
-      test: ["CMD-SHELL", "wget --no-verbose --tries=1 -O - http://127.0.0.1:8123/ping || exit 1"]
-      start_period: 1m
 
   plausible:
-    image: ghcr.io/plausible/community-edition:v2.1.5
+    image: ghcr.io/plausible/community-edition:v2.1.1
     stop_grace_period: 60s
     restart: always
-    command: sh -c "/entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
+    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
     depends_on:
-      plausible_db:
-        condition: service_healthy
-      plausible_events_db:
-        condition: service_healthy
-    volumes:
-      - ./data/plausible:/var/lib/plausible
-    ulimits:
-      nofile:
-        soft: 65535
-        hard: 65535
-    environment:
-      - TMPDIR=/var/lib/plausible/tmp
-      # required: https://github.com/plausible/community-edition/wiki/configuration#required
-      - BASE_URL=${BASE_URL}
-      - SECRET_KEY_BASE=${SECRET_KEY_BASE}
-      # optional: https://github.com/plausible/community-edition/wiki/configuration#optional
-      # registration: https://github.com/plausible/community-edition/wiki/configuration#registration
-      - TOTP_VAULT_KEY
-      - DISABLE_REGISTRATION
-      - ENABLE_EMAIL_VERIFICATION
-      # web: https://github.com/plausible/community-edition/wiki/configuration#web
-      - HTTP_PORT
-      - HTTPS_PORT
-      # databases: https://github.com/plausible/community-edition/wiki/configuration#database
-      - DATABASE_URL
-      - CLICKHOUSE_DATABASE_URL
-      # Google: https://github.com/plausible/community-edition/wiki/configuration#google
-      - GOOGLE_CLIENT_ID
-      - GOOGLE_CLIENT_SECRET
-      # geolocation: https://github.com/plausible/community-edition/wiki/configuration#ip-geolocation
-      - IP_GEOLOCATION_DB
-      - GEONAMES_SOURCE_FILE
-      - MAXMIND_LICENSE_KEY
-      - MAXMIND_EDITION
-      # email: https://github.com/plausible/community-edition/wiki/configuration#email
-      - MAILER_ADAPTER
-      - MAILER_EMAIL
-      - MAILER_NAME
-      - SMTP_HOST_ADDR
-      - SMTP_HOST_PORT
-      - SMTP_USER_NAME
-      - SMTP_USER_PWD
-      - SMTP_HOST_SSL_ENABLED
-      - POSTMARK_API_KEY
-      - MAILGUN_API_KEY
-      - MAILGUN_DOMAIN
-      - MAILGUN_BASE_URI
-      - MANDRILL_API_KEY
-      - SENDGRID_API_KEY
+      - plausible_db
+      - plausible_events_db
+    ports:
+      - 127.0.0.1:84:8000
+    env_file:
+      - .env

plausible-conf.env

@@ -0,0 +1,3 @@
+BASE_URL=replace-me
+SECRET_KEY_BASE=replace-me
+TOTP_VAULT_KEY=replace-me

reverse-proxy/README.md

@@ -0,0 +1,47 @@
+This directory contains pre-made configurations for various reverse proxies. Which flavor you should choose depends on your setup.
+
+## No existing reverse proxy
+
+If you aren't running an existing reverse proxy, then you can use the [`caddy-gen`](https://github.com/wemake-services/caddy-gen) based docker-compose file. Update it to include the domain name you use for your server, then combine it with the existing docker-compose files:
+
+```shell
+$ docker-compose -f docker-compose.yml -f reverse-proxy/docker-compose.caddy-gen.yml up
+```
+
+## Existing reverse proxy
+
+If you are already running a reverse proxy, then the above will not work as it will clash with the existing port bindings. You should instead use one of the available configuration files:
+
+### NGINX
+
+If you already have NGINX running as a system service, use the configuration file in the `nginx` directory.
+
+Edit the file `reverse-proxy/nginx/plausible` to contain the domain name you use for your server, then copy it into NGINX's configuration folder. Enable it by creating a symlink in NGINX's enabled sites folder. Finally use Certbot to create a TLS certificate for your site:
+
+```shell
+$ sudo cp reverse-proxy/nginx/plausible /etc/nginx/sites-available
+$ sudo ln -s /etc/nginx/sites-available/plausible /etc/nginx/sites-enabled/plausible
+$ sudo certbot --nginx
+```
+
+### Traefik 2
+
+If you already have a Traefik container running on Docker, use the docker-compose file in the `traefik` directory. Note that it assumes that your Traefik container is set up to support certificate generation.
+
+Edit the file `reverse-proxy/traefik/docker-compose.traefik.yml` to contain the domain name you use for your server, then combine it with the existing docker-compose files:
+
+```shell
+$ docker-compose -f docker-compose.yml -f reverse-proxy/traefik/docker-compose.traefik.yml up
+```
+
+### Apache2
+Install the necessary Apache modules and restart Apache. Edit the file `reverse-proxy/apache2/plausible.conf` to contain the domain name you use for your server, then copy it into Apache's configuration folder. Enable it by creating a symlink in Apache's enabled sites folder with `a2ensite` command. Finally use Certbot to create a TLS certificate for your site:
+
+```shell
+$ sudo a2enmod proxy proxy_http proxy_ajp remoteip headers proxy_wstunnel
+$ sudo systemctl restart apache2
+$ sudo cp reverse-proxy/apache2/plausible.conf /etc/apache2/sites-available/
+$ sudo a2ensite plausible.conf
+$ sudo systemctl restart apache2
+$ sudo certbot --apache
+```

reverse-proxy/apache2/plausible.conf

@@ -0,0 +1,20 @@
+<VirtualHost *:80>
+
+ ServerAdmin admin@example.com
+ ServerName example.com
+
+ ProxyPreserveHost On
+ ProxyAddHeaders On
+ ProxyPassMatch ^/(live/websocket)$  ws://localhost:8000/$1
+ ProxyPass / http://localhost:8000/
+ ProxyPassReverse / http://localhost:8000/
+
+ SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" forwarded
+
+ ErrorLog ${APACHE_LOG_DIR}/error.log
+ CustomLog ${APACHE_LOG_DIR}/access.log combined env=!forwarded
+ CustomLog ${APACHE_LOG_DIR}/access.log forwarded env=forwarded
+    
+</VirtualHost>

reverse-proxy/docker-compose.caddy-gen.yml

@@ -0,0 +1,24 @@
+version: "3.3"
+services:
+  caddy-gen:
+    container_name: caddy-gen
+    image: "wemakeservices/caddy-gen:latest"
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - caddy-certificates:/data/caddy
+    ports:
+      - "80:80"
+      - "443:443"
+    depends_on:
+      - plausible
+
+  plausible:
+    labels:
+      virtual.host: "example.com" # change to your domain name
+      virtual.port: "8000"
+      virtual.tls-email: "admin@example.com" # change to your email
+        
+volumes:
+    caddy-certificates:
+        driver: local

reverse-proxy/nginx/plausible

@@ -0,0 +1,19 @@
+server {
+	# replace example.com with your domain name
+	server_name example.com;
+	
+	listen 80;
+	listen [::]:80;
+
+	location / {
+		proxy_pass http://127.0.0.1:8000;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+		location = /live/websocket {
+			proxy_pass http://127.0.0.1:8000;
+			proxy_http_version 1.1;
+			proxy_set_header Upgrade $http_upgrade;
+			proxy_set_header Connection "Upgrade";
+		}
+	}
+}

reverse-proxy/traefik/docker-compose.traefik.yml

@@ -0,0 +1,8 @@
+version: "3.3"
+services:
+  plausible:
+    labels:
+      traefik.enable: "true"
+      traefik.http.routers.plausible.rule: "Host(`example.com`)" # change to your domain name
+      traefik.http.routers.plausible.entrypoints: "websecure"
+      traefik.http.services.plausible.loadbalancer.server.port: "8000"

upgrade/postgres.md

@@ -0,0 +1,159 @@
+Guide to upgrading PostgreSQL version `>= 12` to version `14` using `pg_dump` and `psql`. Based on [Upgrade a PostgreSQL database with docker.](https://hollo.me/devops/upgrade-postgresql-database-with-docker.html)
+
+**Note:** following this guide you'd need to stop some containers and it would make your plausible instance temporarily unavailable.
+
+---
+
+### Plan
+
+1. dump contents of the old version PostgreSQL to a file
+1. copy the dump to the host
+1. replace old version PostgreSQL with new version PostgreSQL
+1. copy and load the dump into new version PostgreSQL
+
+---
+
+### Steps
+
+1. Stop `plausible` to avoid writing to old `plausible_db`
+
+```console
+> docker compose stop plausible
+[+] Running 2/2
+ ⠿ Container hosting-plausible-1     Stopped           6.5s
+```
+
+2. Dump old `plausible_db` contents to a backup file
+
+```console
+> docker compose exec plausible_db sh -c "pg_dump -U postgres plausible_db > plausible_db.bak"
+```
+
+3. Copy the backup to the host
+
+```console
+> docker compose cp plausible_db:plausible_db.bak plausible_db.bak
+```
+
+4. (Optional) verify backup went OK
+
+```console
+> head plausible_db.bak
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 12.12 (Debian 12.12-1.pgdg110+1)
+-- Dumped by pg_dump version 12.12 (Debian 12.12-1.pgdg110+1)
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+```
+
+5. Edit `docker-compose.yml` to use new PostgreSQL version, here we update from `v12` to `v14`, alpine flavour.
+
+```diff
+  plausible_db:
+-   image: postgres:12
++   image: postgres:14-alpine
+    restart: always
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=postgres
+```
+
+6. Ensure relevant containers are stopped
+
+```console
+> docker compose stop plausible plausible_db
+[+] Running 2/2
+ ⠿ Container hosting-plausible-1     Stopped           0.0s
+ ⠿ Container hosting-plausible_db-1  Stopped           0.2s
+```
+
+7. Remove old `plausible_db` container to be able to nuke its volume in the next step
+
+```console
+> docker compose rm plausible_db
+? Going to remove hosting-plausible_db-1 Yes
+[+] Running 1/0
+ ⠿ Container hosting-plausible_db-1  Removed           0.0s
+```
+
+8. Remove old `plausible_db` volume, mine is named `hosting_db-data`
+
+```console
+> docker volume ls
+DRIVER    VOLUME NAME
+<...snip...>
+local     hosting_db-data
+local     hosting_event-data
+<...snip...>
+
+> docker volume rm hosting_db-data
+hosting_db-data
+```
+
+9. Start new version `plausible_db` container
+
+```console
+> docker compose up plausible_db -d
+[+] Running 9/9
+ ⠿ plausible_db Pulled                                 9.3s
+   ⠿ 9b18e9b68314 Already exists                       0.0s
+   ⠿ 75aada9edfc5 Pull complete                        1.2s
+   ⠿ 820773693750 Pull complete                        1.2s
+   ⠿ 8812bb04ef2e Pull complete                        5.2s
+   ⠿ 2ccec0f7805c Pull complete                        5.2s
+   ⠿ 833f9b98598e Pull complete                        5.3s
+   ⠿ 1eb578dc04e6 Pull complete                        5.4s
+   ⠿ c873bf6204df Pull complete                        5.4s
+[+] Running 2/2
+ ⠿ Volume "hosting_db-data"          Created           0.0s
+ ⠿ Container hosting-plausible_db-1  Started           0.5s
+```
+
+10. Create new DB and load data into it
+
+```console
+> docker compose exec plausible_db createdb -U postgres plausible_db
+> docker compose cp plausible_db.bak plausible_db:plausible_db.bak
+
+> docker compose exec plausible_db sh -c "psql -U postgres -d plausible_db < plausible_db.bak"
+SET
+SET
+SET
+SET
+SET
+ set_config
+------------
+
+(1 row)
+
+SET
+SET
+SET
+SET
+CREATE EXTENSION
+<...snip...>
+```
+
+11. Start all other containers
+
+```console
+> docker compose up -d
+[+] Running 4/4
+ ⠿ Container hosting-plausible_events_db-1  Running           0.0s
+ ⠿ Container hosting-mail-1                 Running           0.0s
+ ⠿ Container hosting-plausible_db-1         Started           0.5s
+ ⠿ Container hosting-plausible-1            Started           0.5s
+```
+
+12. (Optional) Remove backups from the container and the host
+
+```console
+> rm plausible_db.bak
+> docker compose exec plausible_db rm plausible_db.bak
+```