- [docker](Dockerfile) Use syslogd, don't write to /var/log/apache2/other_vhosts_access.log.

- [docker](Dockerfile) No need for python to write bytecode to container. Disabling that. - [repo](src) Separate source code in by which service it belongs to. - [kopano](src/kopano) Configure kopano-spamd. - [kopano](src/kopano) Workaround kopano-spamd bug: /var/lib/kopano/spamd/ham created with wrong permissions.
2020-02-04 13:38:06 +01:00 · 2020-02-04 13:38:06 +01:00 · 40c0b85046
parent 8bc08ee49e
commit 40c0b85046
13 changed files with 122 additions and 34 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,3 +1,11 @@
+# 1.1.6
+
+- [docker](Dockerfile) Use syslogd, don't write to /var/log/apache2/other_vhosts_access.log.
+- [docker](Dockerfile) No need for python to write bytecode to container. Disabling that.
+- [repo](src) Separate source code in by which service it belongs to.
+- [kopano](src/kopano) Configure kopano-spamd.
+- [kopano](src/kopano) Workaround kopano-spamd bug: /var/lib/kopano/spamd/ham created with wrong permissions.
+
 # 1.1.5

 - [demo](demo) Use host timezone by mounting /etc/localtime.
--- a/11
+++ b/11
@ -8,6 +8,7 @@ ARG	ARCH
 FROM	${ARCH:+$ARCH/}$DIST:$REL AS base
 LABEL	maintainer=mlan
 ENV	DEBIAN_FRONTEND=noninteractive \
+	PYTHONDONTWRITEBYTECODE=PleaseNoPyCache \
 	DOCKER_BIN_DIR=/usr/local/bin \
 	DOCKER_RUNSV_DIR=/etc/service \
 	DOCKER_ENTRY_DIR=/etc/entrypoint.d \
@ -18,7 +19,7 @@ ENV	DEBIAN_FRONTEND=noninteractive \
 	DOCKER_BUILD_DEB_DIR=/tmp/deb \
 	DOCKER_BUILD_PASSES=1 \
 	SYSLOG_OPTIONS='-S' \
-	SYSLOG_LEVEL=4
+	SYSLOG_LEVEL=5
 #
 # Copy utility scripts including entrypoint.sh to image
 #
@ -79,6 +80,8 @@ ENV	DEBIAN_FRONTEND=noninteractive \
 	DOCKER_RUNSV_DIR=/etc/service \
 	DOCKER_BIN_DIR=/usr/local/bin \
 	LMTP_LISTEN=*:2003 \
+	SA_GROUP=kopano \
+	LOG_METHOD=syslog \
 	DOCKER_BUILD_DEB_DIR=/tmp/deb \
 	DOCKER_BUILD_PASSES=1
 #
@ -102,12 +105,12 @@ RUN	mkdir -p $DOCKER_BUILD_DEB_DIR \
 	"-f kopano-search -F" \
 	"kopano-server -F" \
 	"kopano-spooler -F" \
+	"kopano-spamd -F" \
 	"-d kopano-grapi serve" \
 	"-d kopano-kapid serve --log-timestamp=false" \
 	"-d kopano-konnectd serve --log-timestamp=false" \
 	"-d kopano-monitor -F" \
-	"-d kopano-presence -F" \
-	"-d kopano-spamd -F"
+	"-d kopano-presence -F"
 #
 # Have runit's runsvdir start all services
 #
@ -163,7 +166,7 @@ RUN	apt-get install --yes --no-install-recommends apache2 libapache2-mod-php \
 	&& mkdir -p /etc/kopano/theme/Custom \
 	&& ln -sf /etc/kopano/theme/Custom /usr/share/kopano-webapp/plugins/. \
 #	&& conf modify /etc/apache2/apache2.conf '^LogLevel' crit \
-#	&& a2disconf other-vhosts-access-log \
+	&& a2disconf other-vhosts-access-log \
 	&& a2dissite 000-default.conf \
 	&& a2ensite kopano-webapp \
 	&& rm -rf $DOCKER_BUILD_DEB_DIR \
--- a/4
+++ b/4
@ -5,7 +5,7 @@ BLD_ARG  ?=
 BLD_REPO ?= mlan/kopano
 BLD_VER  ?= latest
 BLD_TGT  ?= full
-SRC_VER  ?= $(shell src/docker/bin/kopano-webaddr.sh -VV)
+SRC_VER  ?= $(shell src/kopano/bin/kopano-webaddr.sh -VV)

 _version  = $(if $(findstring $(BLD_TGT),$(1)),$(2),$(if $(findstring latest,$(2)),$(1),$(1)-$(2)))

@ -27,7 +27,7 @@ build_%: Dockerfile
 	-t $(BLD_REPO):$(call _version,$*,$(SRC_VER)) .

 version:
-	src/docker/bin/kopano-webaddr.sh -VV
+	src/kopano/bin/kopano-webaddr.sh -VV

 prune:
 	docker image prune
--- a/README.md
+++ b/README.md
@ -2,7 +2,7 @@

 ![travis-ci test](https://img.shields.io/travis/mlan/docker-kopano.svg?label=build&style=popout-square&logo=travis)
 ![docker build](https://img.shields.io/docker/cloud/build/mlan/kopano.svg?label=build&style=popout-square&logo=docker)
-![image Size](https://img.shields.io/microbadger/image-size/mlan/kopano.svg?label=size&style=popout-square&logo=docker)
+![image Size](https://img.shields.io/docker/image-size/mlan/kopano.svg?label=size&style=popout-square&logo=docker)
 ![docker stars](https://img.shields.io/docker/stars/mlan/kopano.svg?label=stars&style=popout-square&logo=docker)
 ![docker pulls](https://img.shields.io/docker/pulls/mlan/kopano.svg?label=pulls&style=popout-square&logo=docker)

@ -80,11 +80,13 @@ services:
      - MYSQL_DATABASE=${MYSQL_DATABASE-kopano}
      - MYSQL_USER=${MYSQL_USER-kopano}
      - MYSQL_PASSWORD=${MYSQL_PASSWORD-secret}
-      - SYSLOG_LEVEL=3
+      - SYSLOG_LEVEL=${SYSLOG_LEVEL-3}
    volumes:
      - mail-conf:/etc/kopano
      - mail-atch:/var/lib/kopano/attachments
      - mail-sync:/var/lib/z-push
+      - mail-spam:/var/lib/kopano/spamd     # kopano-spamd integration
+      - /etc/localtime:/etc/localtime:ro    # Use host timezone

  mail-mta:
    image: mlan/postfix-amavis
@ -96,18 +98,14 @@ services:
    depends_on:
      - auth
    environment:
-      - MESSAGE_SIZE_LIMIT=${MESSAGE_SIZE_LIMIT-25600000}
      - LDAP_HOST=auth
      - VIRTUAL_TRANSPORT=lmtp:mail-app:2003
-      - SMTP_RELAY_HOSTAUTH=${SMTP_RELAY_HOSTAUTH-}
-      - SMTP_TLS_SECURITY_LEVEL=${SMTP_TLS_SECURITY_LEVEL-}
-      - SMTP_TLS_WRAPPERMODE=${SMTP_TLS_WRAPPERMODE-no}
      - LDAP_USER_BASE=ou=${LDAP_USEROU-users},${LDAP_BASE-dc=example,dc=com}
      - LDAP_QUERY_FILTER_USER=(&(objectclass=${LDAP_USEROBJ-posixAccount})(mail=%s))
-      - DKIM_SELECTOR=${DKIM_SELECTOR-default}
-      - SYSLOG_LEVEL=4
    volumes:
      - mail-mta:/srv
+      - mail-spam:/var/lib/kopano/spamd     # kopano-spamd integration
+      - /etc/localtime:/etc/localtime:ro    # Use host timezone

  mail-db:
    image: mariadb
@ -122,6 +120,7 @@ services:
      - MYSQL_PASSWORD=${MYSQL_PASSWORD-secret}
    volumes:
      - mail-db:/var/lib/mysql
+      - /etc/localtime:/etc/localtime:ro    # Use host timezone

  auth:
    image: mlan/openldap
@ -131,6 +130,7 @@ services:
      - LDAP_LOGLEVEL=parse
    volumes:
      - auth-db:/srv
+      - /etc/localtime:/etc/localtime:ro    # Use host timezone

 networks:
  backend:
@ -141,10 +141,11 @@ volumes:
  mail-atch:
  mail-db:
  mail-mta:
+  mail-spam:
  mail-sync:
 ```

-This repository contains a [demo](demo) directory which hold the [docker-compose.yml](demo/docker-compose.yml) file as well as a [Makefile](demo/Makefile) which might come handy. From within the [demo](demo) directory you can start the `mlan/kopano` container simply by typing:
+This repository contains a [demo](demo) directory which hold the [docker-compose.yml](demo/docker-compose.yml) file as well as a [Makefile](demo/Makefile) which might come handy. From within the [demo](demo) directory you can start the containers by typing:

 ```bash
 make init
@ -166,7 +167,7 @@ To see all available configuration variables you can run `man` within the contai
 make mail-app-man_server
 ```

-If you do, you will notice that configuration variable names are all lower case, but they will be matched with all uppercase environment variables by the container entrypoint script. 
+If you do, you will notice that configuration variable names are all lower case, but they will be matched with all uppercase environment variables by the container `entrypoint.sh` script. 

 ## SQL database configuration

@ -252,9 +253,19 @@ Hint: Use the `kopanoAccount` attribute in the filter to differentiate between n

 By default the `imap` and `pop3` services are disabled for all users. You can set the environment variable `DISABLED_FEATURES=` to enable both `imap` and `pop3`. In this list you can disable certain features for users. This list is space separated, and currently may contain the following features: `imap`, `pop3`. Default: `DISABLED_FEATURES=imap pop3`

-### Logging `LOG_LEVEL`
+## Logging `SYSLOG_LEVEL`, `LOG_LEVEL`

-The level of output for logging in the range from 0 to 6. 0 means no logging, 1 for critical messages only, 2 for error or worse, 3 for warning or worse, 4 for notice or worse, 5 for info or worse, 6 debug. Default: `LOG_LEVEL=3`
+The level of output for logging is in the range from 0 to 7. The default is: `SYSLOG_LEVEL=5`.
+
+| emerg | alert | crit | err  | warning | notice | info | debug |
+| ----- | ----- | ---- | ---- | ------- | ------ | ---- | ----- |
+| 0     | 1     | 2    | 3    | 4       | **5**  | 6    | 7     |
+
+Separately, `LOG_LEVEL` controls the logging level of the Kopano services. `LOG_LEVEL` takes valued from 0 to 6, where the default is `LOG_LEVEL=3`.
+
+| none | crit | err  | warning | notice | info | debug |
+| ---- | ---- | ---- | ------- | ------ | ---- | ----- |
+| 0    | 1    | 2    | **3**   | 4      | 5    | 6     |

 ## Custom themes

@ -287,3 +298,16 @@ TCP Port number used to contact the `SMTP_SERVER`. Default: `SMTP_PORT=25`
 ### Configuring postfix

 The Kopano server listens to the port 2003 and expect the [LMTP](https://en.wikipedia.org/wiki/Local_Mail_Transfer_Protocol) protocol. For Postfix you can define `VIRTUAL_TRANSPORT=lmtp:mail-app:2003` assuming the `mlan/kopano` container is named `mail-app`
+
+## Kopano-spamd integration with [mlan/postfix-amavis](https://github.com/mlan/docker-postfix-amavis)
+
+[Kopano-spamd](https://kb.kopano.io/display/WIKI/Kopano-spamd) allow users to
+drag messages into the Junk folder triggering the anti-spam filter to learn it as spam. If the user moves the message back to the inbox,
+the anti-spam filter will unlearn it.
+
+To allow kopano-spamd integration the kopano and postfix-amavis containers need to
+share the `/var/lib/kopano/spamd` folder. If this directory exists within the
+postfix-amavis container, the spamd-spam and spamd-ham service will be started.
+They will run `sa-learn --spam` or `sa-learn --ham`,
+respectively when a message is placed in either `var/lib/kopano/spamd/spam` or
+`var/lib/kopano/spamd/ham`.
--- a/ROADMAP.md
+++ b/ROADMAP.md
@ -1,19 +1,14 @@
 # Road map

-## kopano_spamd
-
-[discussion](https://jira.kopano.io/browse/KC-666)
-
- let kopano-spamd create /var/lib/kopano/spamd/{ham,spam} with perm 770, user kopano, group amavis/spamassassin
- instead of invoking sa-learn, kopano-spamd should just write to ham  or spam folder depending on what happens (move to spam, spam, move from  spam, ham)
- create a simple python script that will use inotify on the ham and  spam directory. Whenever a new file appear then run sa-learn --spam/–ham  and delete the file on success.
-
-So let the Kopano and postfix containers share the `var/lib/kopano/spamd` folder and run the cron job in the postfix container.
-
 ## Revisit Persistent Data

 Consider consolidating directories which are candidates for persistence under `/srv`.

+### Kopano Search
+
+The kopano-search module keeps its database here, /var/lib/kopano/search.
+Consider to also consolidating it under /srv to simplify making it persistent?
+
 ## Improve Health Check?

 Verify the user anonymously.
--- a/demo/.env
+++ b/demo/.env
@ -1,8 +1,12 @@
 COMPOSE_PROJECT_NAME=demo
-SYSLOG_LEVEL=7
+SYSLOG_LEVEL=5
 MAIL_DOMAIN=example.com
 MAIL_SRV=mx
 DKIM_SELECTOR=default
+SA_TAG_LEVEL_DEFLT=-999
+SA_DEBUG=0
+LOG_LEVEL=5
+RAZOR_REGISTRATION=
 LDAP_BASE=dc=example,dc=com
 LDAP_USEROU=users
 LDAP_USEROBJ=posixAccount
--- a/demo/Makefile
+++ b/demo/Makefile
@ -12,7 +12,7 @@ variables:
 ps:
 	docker-compose ps

-init: auth-up wait_11 auth-mod_index auth-add_user mail-db-up mail-mta-up wait_12 mail-app-up wait_23 mail-app-create_store
+init: auth-up wait_11 auth-mod_index auth-add_user mail-db-up mail-mta-up mail-app-up wait_93 mail-app-create_store

 up:
 	docker-compose up -d
@ -26,6 +26,9 @@ destroy:
 config:
 	docker-compose config

+logs:
+	docker-compose logs --tail 10
+
 wait_%:
 	sleep $*

@ -91,6 +94,9 @@ mail-mta-apk_list:
 mail-mta-quarantine_list:
 	docker-compose exec mail-mta amavisd-ls

+mail-mta-freshclam_nodns:
+	docker-compose exec mail-mta freshclam --no-dns
+
 mail-mta-debugtools:
 	docker-compose exec mail-mta apk --no-cache --update add \
 	nano less lsof htop openldap-clients bind-tools iputils strace
--- a/demo/docker-compose.yml
+++ b/demo/docker-compose.yml
@ -27,6 +27,7 @@ services:
      - mail-conf:/etc/kopano
      - mail-atch:/var/lib/kopano/attachments
      - mail-sync:/var/lib/z-push
+      - mail-spam:/var/lib/kopano/spamd     # kopano-spamd integration
      - /etc/localtime:/etc/localtime:ro    # Use host timezone
    cap_add: # helps debugging by alowing strace
      - sys_ptrace
@ -57,6 +58,7 @@ services:
      - RAZOR_REGISTRATION=${RAZOR_REGISTRATION-}
    volumes:
      - mail-mta:/srv
+      - mail-spam:/var/lib/kopano/spamd     # kopano-spamd integration
      - /etc/localtime:/etc/localtime:ro    # Use host timezone
    cap_add: # helps debugging by alowing strace
      - sys_ptrace
@ -95,4 +97,5 @@ volumes:
  mail-atch:
  mail-db:
  mail-mta:
+  mail-spam:
  mail-sync:
--- a/hooks/post_push
+++ b/hooks/post_push
@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # hooks/post_push

-version=$(src/docker/bin/kopano-webaddr.sh -VV)
+version=$(src/kopano/bin/kopano-webaddr.sh -VV)

 _docker_add_tags() {
 	local tag="$1"
--- a/src/kopano/bin/kopano-webaddr.sh
+++ b/src/kopano/bin/kopano-webaddr.sh
--- a/src/kopano/entrypoint.d/10_fix_spamd_ham
+++ b/src/kopano/entrypoint.d/10_fix_spamd_ham
@ -0,0 +1,41 @@
+#!/bin/sh
+#
+# 10_fix_spamd_ham
+#
+# kopano-spamd creates /var/lib/kopano/spamd/ham with root ownership, 
+# but needs to be kopano
+#
+
+#
+# config
+#
+
+DOCKER_USER=${DOCKER_USER-kopano}
+DOCKER_SPAMD_DIR=${DOCKER_SPAMD_DIR-/var/lib/kopano/spamd}
+
+#
+# define helpers
+#
+
+inform() { printf "entrypoint[$$]: INFO:$(basename $0): $*.\n" ;}
+
+fixattr() {
+	for dir in $@; do
+		if [ -n "$(find $dir ! -user $DOCKER_USER -print -exec chown -h $DOCKER_USER: {} \;)" ]; then
+			inform "Changed owner to $DOCKER_USER for some files in $dir"
+		fi
+		if [ -n "$(find -L $dir ! -user $DOCKER_USER -print -exec chown $DOCKER_USER: {} \;)" ]; then
+			inform "Changed owner to $DOCKER_USER for some files in $dir"
+		fi
+		if [ -n "$(find -H $dir ! -perm -u+rw -print -exec chmod u+rw {} \;)" ]; then
+			inform "Changed permission to rw for some files in $dir"
+		fi
+	done
+}
+
+#
+# run
+#
+
+mkdir -p $DOCKER_SPAMD_DIR/ham
+fixattr $DOCKER_SPAMD_DIR/ham
--- a/src/kopano/entrypoint.d/20_fix_attr
+++ b/src/kopano/entrypoint.d/20_fix_attr
@ -11,6 +11,7 @@

 DOCKER_USER=${DOCKER_USER-kopano}
 DOCKER_ATCH_DIR=${DOCKER_ATCH_DIR-/var/lib/kopano/attachments}
+DOCKER_SPAMD_DIR=${DOCKER_SPAMD_DIR-/var/lib/kopano/spamd}

 #
 # define helpers
@ -36,4 +37,4 @@ fixattr() {
 # run
 #

-fixattr $DOCKER_ATCH_DIR
+fixattr $DOCKER_ATCH_DIR $DOCKER_SPAM_DIR
--- a/src/kopano/entrypoint.d/50_update-config
+++ b/src/kopano/entrypoint.d/50_update-config
@ -17,6 +17,7 @@ server_cfg_file=$DOCKER_CONF_DIR1/server.cfg
 ldap_cfg_file=$DOCKER_CONF_DIR1/ldap.cfg
 spooler_cfg_file=$DOCKER_CONF_DIR1/spooler.cfg
 dagent_cfg_file=$DOCKER_CONF_DIR1/dagent.cfg
+spamd_cfg_file=$DOCKER_CONF_DIR1/spamd.cfg

 zpush_cfg_file=$DOCKER_CONF_DIR2/config.php
 sqlstate_cfg_file=$DOCKER_CONF_DIR2/backend/sqlstatemachine/config.php
@ -27,8 +28,9 @@ sqlstate_cfg_file=$DOCKER_CONF_DIR2/backend/sqlstatemachine/config.php

 server_env_vars="MYSQL_HOST MYSQL_PORT MYSQL_DATABASE MYSQL_USER MYSQL_PASSWORD DISABLED_FEATURES USER_PLUGIN LOG_LEVEL"
 ldap_env_vars="LDAP_URI LDAP_HOST LDAP_PORT LDAP_PROTOCOL LDAP_SEARCH_BASE LDAP_USER_TYPE_ATTRIBUTE_VALUE LDAP_GROUP_TYPE_ATTRIBUTE_VALUE LDAP_USER_SEARCH_FILTER"
-spooler_env_vars="SMTP_SERVER SMTP_PORT"
-dagent_env_vars="LMTP_LISTEN"
+spooler_env_vars="SMTP_SERVER SMTP_PORT LOG_LEVEL"
+dagent_env_vars="LMTP_LISTEN LOG_LEVEL"
+spamd_env_vars="LOG_METHOD SPAM_DIR SPAM_DB HEADER_TAG LEARN_HAM HAM_DIR SA_GROUP"

 zpush_env_vars="TIMEZONE USE_CUSTOM_REMOTE_IP_HEADER USE_FULLEMAIL_FOR_LOGIN STATE_MACHINE STATE_DIR LOGBACKEND LOGLEVEL LOGAUTHFAIL LOG_SYSLOG_PROGRAM LOG_SYSLOG_FACILITY SYNC_CONFLICT_DEFAULT PING_INTERVAL FILEAS_ORDER SYNC_MAX_ITEMS UNSET_UNDEFINED_PROPERTIES ALLOW_WEBSERVICE_USERS_ACCESS USE_PARTIAL_FOLDERSYNC"
 sqlstate_env_vars="STATE_SQL_ENGINE STATE_SQL_SERVER STATE_SQL_PORT STATE_SQL_DATABASE STATE_SQL_USER STATE_SQL_PASSWORD STATE_SQL_OPTIONS"
@ -73,6 +75,7 @@ kopano_cfg() {
 	_kopano_cfg_gen $ldap_cfg_file     $ldap_env_vars
 	_kopano_cfg_gen $spooler_cfg_file  $spooler_env_vars
 	_kopano_cfg_gen $dagent_cfg_file   $dagent_env_vars
+	_kopano_cfg_gen $spamd_cfg_file    $spamd_env_vars
 }

 php_cfg() {