diff --git a/CHANGELOG.md b/CHANGELOG.md index cbb7273..f757606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# 1.1.6 + +- [docker](Dockerfile) Use syslogd, don't write to /var/log/apache2/other_vhosts_access.log. +- [docker](Dockerfile) No need for python to write bytecode to container. Disabling that. +- [repo](src) Separate source code in by which service it belongs to. +- [kopano](src/kopano) Configure kopano-spamd. +- [kopano](src/kopano) Workaround kopano-spamd bug: /var/lib/kopano/spamd/ham created with wrong permissions. + # 1.1.5 - [demo](demo) Use host timezone by mounting /etc/localtime. diff --git a/Dockerfile b/Dockerfile index 8d8cbe9..0ea90a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ ARG ARCH FROM ${ARCH:+$ARCH/}$DIST:$REL AS base LABEL maintainer=mlan ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=PleaseNoPyCache \ DOCKER_BIN_DIR=/usr/local/bin \ DOCKER_RUNSV_DIR=/etc/service \ DOCKER_ENTRY_DIR=/etc/entrypoint.d \ @@ -18,7 +19,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ DOCKER_BUILD_DEB_DIR=/tmp/deb \ DOCKER_BUILD_PASSES=1 \ SYSLOG_OPTIONS='-S' \ - SYSLOG_LEVEL=4 + SYSLOG_LEVEL=5 # # Copy utility scripts including entrypoint.sh to image # @@ -79,6 +80,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ DOCKER_RUNSV_DIR=/etc/service \ DOCKER_BIN_DIR=/usr/local/bin \ LMTP_LISTEN=*:2003 \ + SA_GROUP=kopano \ + LOG_METHOD=syslog \ DOCKER_BUILD_DEB_DIR=/tmp/deb \ DOCKER_BUILD_PASSES=1 # @@ -102,12 +105,12 @@ RUN mkdir -p $DOCKER_BUILD_DEB_DIR \ "-f kopano-search -F" \ "kopano-server -F" \ "kopano-spooler -F" \ + "kopano-spamd -F" \ "-d kopano-grapi serve" \ "-d kopano-kapid serve --log-timestamp=false" \ "-d kopano-konnectd serve --log-timestamp=false" \ "-d kopano-monitor -F" \ - "-d kopano-presence -F" \ - "-d kopano-spamd -F" + "-d kopano-presence -F" # # Have runit's runsvdir start all services # @@ -163,7 +166,7 @@ RUN apt-get install --yes --no-install-recommends apache2 libapache2-mod-php \ && mkdir -p /etc/kopano/theme/Custom \ && ln -sf /etc/kopano/theme/Custom /usr/share/kopano-webapp/plugins/. \ # && conf modify /etc/apache2/apache2.conf '^LogLevel' crit \ -# && a2disconf other-vhosts-access-log \ + && a2disconf other-vhosts-access-log \ && a2dissite 000-default.conf \ && a2ensite kopano-webapp \ && rm -rf $DOCKER_BUILD_DEB_DIR \ diff --git a/Makefile b/Makefile index 9294475..89991ff 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BLD_ARG ?= BLD_REPO ?= mlan/kopano BLD_VER ?= latest BLD_TGT ?= full -SRC_VER ?= $(shell src/docker/bin/kopano-webaddr.sh -VV) +SRC_VER ?= $(shell src/kopano/bin/kopano-webaddr.sh -VV) _version = $(if $(findstring $(BLD_TGT),$(1)),$(2),$(if $(findstring latest,$(2)),$(1),$(1)-$(2))) @@ -27,7 +27,7 @@ build_%: Dockerfile -t $(BLD_REPO):$(call _version,$*,$(SRC_VER)) . version: - src/docker/bin/kopano-webaddr.sh -VV + src/kopano/bin/kopano-webaddr.sh -VV prune: docker image prune diff --git a/README.md b/README.md index f2b2de1..37503aa 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![travis-ci test](https://img.shields.io/travis/mlan/docker-kopano.svg?label=build&style=popout-square&logo=travis) ![docker build](https://img.shields.io/docker/cloud/build/mlan/kopano.svg?label=build&style=popout-square&logo=docker) -![image Size](https://img.shields.io/microbadger/image-size/mlan/kopano.svg?label=size&style=popout-square&logo=docker) +![image Size](https://img.shields.io/docker/image-size/mlan/kopano.svg?label=size&style=popout-square&logo=docker) ![docker stars](https://img.shields.io/docker/stars/mlan/kopano.svg?label=stars&style=popout-square&logo=docker) ![docker pulls](https://img.shields.io/docker/pulls/mlan/kopano.svg?label=pulls&style=popout-square&logo=docker) @@ -80,11 +80,13 @@ services: - MYSQL_DATABASE=${MYSQL_DATABASE-kopano} - MYSQL_USER=${MYSQL_USER-kopano} - MYSQL_PASSWORD=${MYSQL_PASSWORD-secret} - - SYSLOG_LEVEL=3 + - SYSLOG_LEVEL=${SYSLOG_LEVEL-3} volumes: - mail-conf:/etc/kopano - mail-atch:/var/lib/kopano/attachments - mail-sync:/var/lib/z-push + - mail-spam:/var/lib/kopano/spamd # kopano-spamd integration + - /etc/localtime:/etc/localtime:ro # Use host timezone mail-mta: image: mlan/postfix-amavis @@ -96,18 +98,14 @@ services: depends_on: - auth environment: - - MESSAGE_SIZE_LIMIT=${MESSAGE_SIZE_LIMIT-25600000} - LDAP_HOST=auth - VIRTUAL_TRANSPORT=lmtp:mail-app:2003 - - SMTP_RELAY_HOSTAUTH=${SMTP_RELAY_HOSTAUTH-} - - SMTP_TLS_SECURITY_LEVEL=${SMTP_TLS_SECURITY_LEVEL-} - - SMTP_TLS_WRAPPERMODE=${SMTP_TLS_WRAPPERMODE-no} - LDAP_USER_BASE=ou=${LDAP_USEROU-users},${LDAP_BASE-dc=example,dc=com} - LDAP_QUERY_FILTER_USER=(&(objectclass=${LDAP_USEROBJ-posixAccount})(mail=%s)) - - DKIM_SELECTOR=${DKIM_SELECTOR-default} - - SYSLOG_LEVEL=4 volumes: - mail-mta:/srv + - mail-spam:/var/lib/kopano/spamd # kopano-spamd integration + - /etc/localtime:/etc/localtime:ro # Use host timezone mail-db: image: mariadb @@ -122,6 +120,7 @@ services: - MYSQL_PASSWORD=${MYSQL_PASSWORD-secret} volumes: - mail-db:/var/lib/mysql + - /etc/localtime:/etc/localtime:ro # Use host timezone auth: image: mlan/openldap @@ -131,6 +130,7 @@ services: - LDAP_LOGLEVEL=parse volumes: - auth-db:/srv + - /etc/localtime:/etc/localtime:ro # Use host timezone networks: backend: @@ -141,10 +141,11 @@ volumes: mail-atch: mail-db: mail-mta: + mail-spam: mail-sync: ``` -This repository contains a [demo](demo) directory which hold the [docker-compose.yml](demo/docker-compose.yml) file as well as a [Makefile](demo/Makefile) which might come handy. From within the [demo](demo) directory you can start the `mlan/kopano` container simply by typing: +This repository contains a [demo](demo) directory which hold the [docker-compose.yml](demo/docker-compose.yml) file as well as a [Makefile](demo/Makefile) which might come handy. From within the [demo](demo) directory you can start the containers by typing: ```bash make init @@ -166,7 +167,7 @@ To see all available configuration variables you can run `man` within the contai make mail-app-man_server ``` -If you do, you will notice that configuration variable names are all lower case, but they will be matched with all uppercase environment variables by the container entrypoint script. +If you do, you will notice that configuration variable names are all lower case, but they will be matched with all uppercase environment variables by the container `entrypoint.sh` script. ## SQL database configuration @@ -252,9 +253,19 @@ Hint: Use the `kopanoAccount` attribute in the filter to differentiate between n By default the `imap` and `pop3` services are disabled for all users. You can set the environment variable `DISABLED_FEATURES=` to enable both `imap` and `pop3`. In this list you can disable certain features for users. This list is space separated, and currently may contain the following features: `imap`, `pop3`. Default: `DISABLED_FEATURES=imap pop3` -### Logging `LOG_LEVEL` +## Logging `SYSLOG_LEVEL`, `LOG_LEVEL` -The level of output for logging in the range from 0 to 6. 0 means no logging, 1 for critical messages only, 2 for error or worse, 3 for warning or worse, 4 for notice or worse, 5 for info or worse, 6 debug. Default: `LOG_LEVEL=3` +The level of output for logging is in the range from 0 to 7. The default is: `SYSLOG_LEVEL=5`. + +| emerg | alert | crit | err | warning | notice | info | debug | +| ----- | ----- | ---- | ---- | ------- | ------ | ---- | ----- | +| 0 | 1 | 2 | 3 | 4 | **5** | 6 | 7 | + +Separately, `LOG_LEVEL` controls the logging level of the Kopano services. `LOG_LEVEL` takes valued from 0 to 6, where the default is `LOG_LEVEL=3`. + +| none | crit | err | warning | notice | info | debug | +| ---- | ---- | ---- | ------- | ------ | ---- | ----- | +| 0 | 1 | 2 | **3** | 4 | 5 | 6 | ## Custom themes @@ -287,3 +298,16 @@ TCP Port number used to contact the `SMTP_SERVER`. Default: `SMTP_PORT=25` ### Configuring postfix The Kopano server listens to the port 2003 and expect the [LMTP](https://en.wikipedia.org/wiki/Local_Mail_Transfer_Protocol) protocol. For Postfix you can define `VIRTUAL_TRANSPORT=lmtp:mail-app:2003` assuming the `mlan/kopano` container is named `mail-app` + +## Kopano-spamd integration with [mlan/postfix-amavis](https://github.com/mlan/docker-postfix-amavis) + +[Kopano-spamd](https://kb.kopano.io/display/WIKI/Kopano-spamd) allow users to +drag messages into the Junk folder triggering the anti-spam filter to learn it as spam. If the user moves the message back to the inbox, +the anti-spam filter will unlearn it. + +To allow kopano-spamd integration the kopano and postfix-amavis containers need to +share the `/var/lib/kopano/spamd` folder. If this directory exists within the +postfix-amavis container, the spamd-spam and spamd-ham service will be started. +They will run `sa-learn --spam` or `sa-learn --ham`, +respectively when a message is placed in either `var/lib/kopano/spamd/spam` or +`var/lib/kopano/spamd/ham`. diff --git a/ROADMAP.md b/ROADMAP.md index aa32fa0..8a2ebac 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,19 +1,14 @@ # Road map -## kopano_spamd - -[discussion](https://jira.kopano.io/browse/KC-666) - -- let kopano-spamd create /var/lib/kopano/spamd/{ham,spam} with perm 770, user kopano, group amavis/spamassassin -- instead of invoking sa-learn, kopano-spamd should just write to ham or spam folder depending on what happens (move to spam, spam, move from spam, ham) -- create a simple python script that will use inotify on the ham and spam directory. Whenever a new file appear then run sa-learn --spam/–ham and delete the file on success. - -So let the Kopano and postfix containers share the `var/lib/kopano/spamd` folder and run the cron job in the postfix container. - ## Revisit Persistent Data Consider consolidating directories which are candidates for persistence under `/srv`. +### Kopano Search + +The kopano-search module keeps its database here, /var/lib/kopano/search. +Consider to also consolidating it under /srv to simplify making it persistent? + ## Improve Health Check? Verify the user anonymously. diff --git a/demo/.env b/demo/.env index f5ae002..517b75a 100644 --- a/demo/.env +++ b/demo/.env @@ -1,8 +1,12 @@ COMPOSE_PROJECT_NAME=demo -SYSLOG_LEVEL=7 +SYSLOG_LEVEL=5 MAIL_DOMAIN=example.com MAIL_SRV=mx DKIM_SELECTOR=default +SA_TAG_LEVEL_DEFLT=-999 +SA_DEBUG=0 +LOG_LEVEL=5 +RAZOR_REGISTRATION= LDAP_BASE=dc=example,dc=com LDAP_USEROU=users LDAP_USEROBJ=posixAccount diff --git a/demo/Makefile b/demo/Makefile index c1e152e..43b2522 100644 --- a/demo/Makefile +++ b/demo/Makefile @@ -12,7 +12,7 @@ variables: ps: docker-compose ps -init: auth-up wait_11 auth-mod_index auth-add_user mail-db-up mail-mta-up wait_12 mail-app-up wait_23 mail-app-create_store +init: auth-up wait_11 auth-mod_index auth-add_user mail-db-up mail-mta-up mail-app-up wait_93 mail-app-create_store up: docker-compose up -d @@ -26,6 +26,9 @@ destroy: config: docker-compose config +logs: + docker-compose logs --tail 10 + wait_%: sleep $* @@ -91,6 +94,9 @@ mail-mta-apk_list: mail-mta-quarantine_list: docker-compose exec mail-mta amavisd-ls +mail-mta-freshclam_nodns: + docker-compose exec mail-mta freshclam --no-dns + mail-mta-debugtools: docker-compose exec mail-mta apk --no-cache --update add \ nano less lsof htop openldap-clients bind-tools iputils strace diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml index 3f25120..823d448 100644 --- a/demo/docker-compose.yml +++ b/demo/docker-compose.yml @@ -27,6 +27,7 @@ services: - mail-conf:/etc/kopano - mail-atch:/var/lib/kopano/attachments - mail-sync:/var/lib/z-push + - mail-spam:/var/lib/kopano/spamd # kopano-spamd integration - /etc/localtime:/etc/localtime:ro # Use host timezone cap_add: # helps debugging by alowing strace - sys_ptrace @@ -57,6 +58,7 @@ services: - RAZOR_REGISTRATION=${RAZOR_REGISTRATION-} volumes: - mail-mta:/srv + - mail-spam:/var/lib/kopano/spamd # kopano-spamd integration - /etc/localtime:/etc/localtime:ro # Use host timezone cap_add: # helps debugging by alowing strace - sys_ptrace @@ -95,4 +97,5 @@ volumes: mail-atch: mail-db: mail-mta: + mail-spam: mail-sync: diff --git a/hooks/post_push b/hooks/post_push index dbe52e8..59d5374 100644 --- a/hooks/post_push +++ b/hooks/post_push @@ -1,7 +1,7 @@ #!/usr/bin/env bash # hooks/post_push -version=$(src/docker/bin/kopano-webaddr.sh -VV) +version=$(src/kopano/bin/kopano-webaddr.sh -VV) _docker_add_tags() { local tag="$1" diff --git a/src/docker/bin/kopano-webaddr.sh b/src/kopano/bin/kopano-webaddr.sh similarity index 100% rename from src/docker/bin/kopano-webaddr.sh rename to src/kopano/bin/kopano-webaddr.sh diff --git a/src/kopano/entrypoint.d/10_fix_spamd_ham b/src/kopano/entrypoint.d/10_fix_spamd_ham new file mode 100755 index 0000000..72434e9 --- /dev/null +++ b/src/kopano/entrypoint.d/10_fix_spamd_ham @@ -0,0 +1,41 @@ +#!/bin/sh +# +# 10_fix_spamd_ham +# +# kopano-spamd creates /var/lib/kopano/spamd/ham with root ownership, +# but needs to be kopano +# + +# +# config +# + +DOCKER_USER=${DOCKER_USER-kopano} +DOCKER_SPAMD_DIR=${DOCKER_SPAMD_DIR-/var/lib/kopano/spamd} + +# +# define helpers +# + +inform() { printf "entrypoint[$$]: INFO:$(basename $0): $*.\n" ;} + +fixattr() { + for dir in $@; do + if [ -n "$(find $dir ! -user $DOCKER_USER -print -exec chown -h $DOCKER_USER: {} \;)" ]; then + inform "Changed owner to $DOCKER_USER for some files in $dir" + fi + if [ -n "$(find -L $dir ! -user $DOCKER_USER -print -exec chown $DOCKER_USER: {} \;)" ]; then + inform "Changed owner to $DOCKER_USER for some files in $dir" + fi + if [ -n "$(find -H $dir ! -perm -u+rw -print -exec chmod u+rw {} \;)" ]; then + inform "Changed permission to rw for some files in $dir" + fi + done +} + +# +# run +# + +mkdir -p $DOCKER_SPAMD_DIR/ham +fixattr $DOCKER_SPAMD_DIR/ham diff --git a/src/docker/entrypoint.d/20_fix_attr b/src/kopano/entrypoint.d/20_fix_attr similarity index 89% rename from src/docker/entrypoint.d/20_fix_attr rename to src/kopano/entrypoint.d/20_fix_attr index ce1c6a7..f4d530b 100755 --- a/src/docker/entrypoint.d/20_fix_attr +++ b/src/kopano/entrypoint.d/20_fix_attr @@ -11,6 +11,7 @@ DOCKER_USER=${DOCKER_USER-kopano} DOCKER_ATCH_DIR=${DOCKER_ATCH_DIR-/var/lib/kopano/attachments} +DOCKER_SPAMD_DIR=${DOCKER_SPAMD_DIR-/var/lib/kopano/spamd} # # define helpers @@ -36,4 +37,4 @@ fixattr() { # run # -fixattr $DOCKER_ATCH_DIR +fixattr $DOCKER_ATCH_DIR $DOCKER_SPAM_DIR diff --git a/src/docker/entrypoint.d/50_update-config b/src/kopano/entrypoint.d/50_update-config similarity index 90% rename from src/docker/entrypoint.d/50_update-config rename to src/kopano/entrypoint.d/50_update-config index 1bafef2..c5e67f8 100755 --- a/src/docker/entrypoint.d/50_update-config +++ b/src/kopano/entrypoint.d/50_update-config @@ -17,6 +17,7 @@ server_cfg_file=$DOCKER_CONF_DIR1/server.cfg ldap_cfg_file=$DOCKER_CONF_DIR1/ldap.cfg spooler_cfg_file=$DOCKER_CONF_DIR1/spooler.cfg dagent_cfg_file=$DOCKER_CONF_DIR1/dagent.cfg +spamd_cfg_file=$DOCKER_CONF_DIR1/spamd.cfg zpush_cfg_file=$DOCKER_CONF_DIR2/config.php sqlstate_cfg_file=$DOCKER_CONF_DIR2/backend/sqlstatemachine/config.php @@ -27,8 +28,9 @@ sqlstate_cfg_file=$DOCKER_CONF_DIR2/backend/sqlstatemachine/config.php server_env_vars="MYSQL_HOST MYSQL_PORT MYSQL_DATABASE MYSQL_USER MYSQL_PASSWORD DISABLED_FEATURES USER_PLUGIN LOG_LEVEL" ldap_env_vars="LDAP_URI LDAP_HOST LDAP_PORT LDAP_PROTOCOL LDAP_SEARCH_BASE LDAP_USER_TYPE_ATTRIBUTE_VALUE LDAP_GROUP_TYPE_ATTRIBUTE_VALUE LDAP_USER_SEARCH_FILTER" -spooler_env_vars="SMTP_SERVER SMTP_PORT" -dagent_env_vars="LMTP_LISTEN" +spooler_env_vars="SMTP_SERVER SMTP_PORT LOG_LEVEL" +dagent_env_vars="LMTP_LISTEN LOG_LEVEL" +spamd_env_vars="LOG_METHOD SPAM_DIR SPAM_DB HEADER_TAG LEARN_HAM HAM_DIR SA_GROUP" zpush_env_vars="TIMEZONE USE_CUSTOM_REMOTE_IP_HEADER USE_FULLEMAIL_FOR_LOGIN STATE_MACHINE STATE_DIR LOGBACKEND LOGLEVEL LOGAUTHFAIL LOG_SYSLOG_PROGRAM LOG_SYSLOG_FACILITY SYNC_CONFLICT_DEFAULT PING_INTERVAL FILEAS_ORDER SYNC_MAX_ITEMS UNSET_UNDEFINED_PROPERTIES ALLOW_WEBSERVICE_USERS_ACCESS USE_PARTIAL_FOLDERSYNC" sqlstate_env_vars="STATE_SQL_ENGINE STATE_SQL_SERVER STATE_SQL_PORT STATE_SQL_DATABASE STATE_SQL_USER STATE_SQL_PASSWORD STATE_SQL_OPTIONS" @@ -73,6 +75,7 @@ kopano_cfg() { _kopano_cfg_gen $ldap_cfg_file $ldap_env_vars _kopano_cfg_gen $spooler_cfg_file $spooler_env_vars _kopano_cfg_gen $dagent_cfg_file $dagent_env_vars + _kopano_cfg_gen $spamd_cfg_file $spamd_env_vars } php_cfg() {