Merge branch 'release/0.2.0' into production

* release/0.2.0:
  Add release version to changelog
  Update documentation
  Move code from gvaweb to webtasks
  Add redis dependency
  Add Docker setup

Dockerfile

@@ -0,0 +1,45 @@
+ARG DEBIAN_RELEASE=buster
+FROM debian:$DEBIAN_RELEASE
+LABEL maintainer="Jan Dittberner <jan@dittberner.info>"
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    build-essential \
+    dumb-init \
+    gettext \
+    git \
+    python3-dev \
+    python3-pip \
+    python3-setuptools \
+    python3-virtualenv \
+    python3-wheel \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*.*
+
+RUN python3 -m pip install --prefix=/usr/local pipenv
+
+ARG GVAGID=2000
+ARG GVAUID=2000
+
+ARG GVAAPP=gvaweb
+
+WORKDIR /srv/$GVAAPP
+
+COPY Pipfile Pipfile.lock /srv/$GVAAPP/
+
+RUN addgroup --gid $GVAGID $GVAAPP ; \
+    adduser --home /home/$GVAAPP --shell /bin/bash --uid $GVAUID --gid $GVAGID --disabled-password --gecos "User for gnuviechadmin component $GVAAPP" $GVAAPP
+
+USER $GVAAPP
+RUN python3 -m virtualenv --python=python3 /home/$GVAAPP/$GVAAPP-venv ; \
+    /home/$GVAAPP/$GVAAPP-venv/bin/python3 -m pip install -U pip ; \
+    VIRTUAL_ENV=/home/$GVAAPP/$GVAAPP-venv pipenv install --deploy --ignore-pipfile --dev
+
+VOLUME /srv/$GVAAPP
+
+COPY gvaweb.sh /srv/
+
+ENTRYPOINT ["dumb-init", "/srv/gvaweb.sh"]

Pipfile

@@ -15,6 +15,7 @@ celery = "*"
 kombu = "*"
 pytz = "*"
 jinja2 = "*"
+redis = "*"
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "4420e939c194bdd75c095801149c37d0eb28c9e3010e20e701ee8d95e81b0a52"
+            "sha256": "19baf5d6f093d84ef837123087771dcd28d324a4b12e901f520374272ebbee75"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -116,6 +116,14 @@
             "index": "pypi",
             "version": "==2019.3"
         },
+        "redis": {
+            "hashes": [
+                "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f",
+                "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833"
+            ],
+            "index": "pypi",
+            "version": "==3.4.1"
+        },
         "vine": {
             "hashes": [
                 "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
@@ -125,10 +133,10 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:12248a63bbdf7548f89cb4c7cda4681e537031eda29c02ea29674bc6854460c2",
+                "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
-                "sha256:7c0f8e91abc0dc07a5068f315c52cb30c66bfbc581e5b50704c8a2f6ebae794a"
+                "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"
             ],
-            "version": "==3.0.0"
+            "version": "==3.1.0"
         }
     },
     "develop": {

docs/changelog.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+* :release:`0.2.0 <2020-03-03>`
+* :support:`-` add Docker setup for lightweight local testing
+* :support:`-` restructure code to have celery in the webtasks module and drop
+  the gvaldap module
+* :support:`-` update documentation to alabaster theme
+
 * :release:`0.1.8 <2020-03-02>`
 * :bug:`-` salt automation was broken due to changed paths and missing Jinja 2
   dependency

docs/code.rst

@@ -7,30 +7,25 @@ gvaweb is implemented as `Celery`_ app.
 .. _Celery: http://www.celeryproject.org/
 
 
-The project module :py:mod:`gvaweb`
+The module :py:mod:`webtasks`
-===================================
+=============================
 
-.. automodule:: gvaweb
+.. automodule:: webtasks
 
 
-:py:mod:`celery <gvaweb.celery>`
+:py:mod:`celery <webtasks.celery>`
---------------------------------
+----------------------------------
 
-.. automodule:: gvaweb.celery
+.. automodule:: webtasks.celery
    :members:
 
 
-:py:mod:`settings <gvaweb.settings>`
+:py:mod:`settings <webtasks.settings>`
-------------------------------------
+--------------------------------------
 
-.. automodule:: gvaweb.settings
+.. automodule:: webtasks.settings
 
 
-:py:mod:`webtasks` app
-======================
-
-.. automodule:: webtasks
-
 :py:mod:`tasks <webtasks.tasks>`
 --------------------------------
 

docs/conf.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
+# pymode:lint_ignore=E501
 #
 # gvaweb documentation build configuration file, created by
 # sphinx-quickstart on Mon Jan 26 15:51:21 2015.
 #
-# This file is execfile()d with the current directory set to its
+# This file is execfile()d with the current directory set to its containing dir.
-# containing dir.
 #
 # Note that not all possible configuration values are present in this
 # autogenerated file.
@@ -30,18 +30,17 @@ os.environ['GVAWEB_WWWUSER_MOUNT'] = '/srv/wwwfiles'
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'
 
-# Add any Sphinx extension module names here, as strings. They can be
+# Add any Sphinx extension module names here, as strings. They can be extensions
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-# ones.
 extensions = ['releases', 'sphinx.ext.autodoc', 'celery.contrib.sphinx']
 
+# configuration for releases extension
+releases_issue_uri = 'https://git.dittberner.info/gnuviech/gvaweb/issues/%s'
+releases_release_uri = 'https://git.dittberner.info/gnuviech/gvaweb/src/tag/%s'
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
-releases_issue_uri = 'https://git.dittberner.info/gnuviech/gvaweb/issues/%s'
-
-releases_release_uri = 'https://git.dittberner.info/gnuviech/gvaweb/src/tag/%s'
-
 # The suffix of source filenames.
 source_suffix = '.rst'
 
@@ -59,10 +58,11 @@ copyright = u'2015-2020, Jan Dittberner'
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-# The short X.Y version.
-version = '0.1'
 # The full version, including alpha/beta/rc tags.
-release = '0.1.8'
+from webtasks import __version__ as release
+
+# The short X.Y version.
+version = ".".join(release.split('.')[:2])
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -78,8 +78,7 @@ release = '0.1.8'
 # directories to ignore when looking for source files.
 exclude_patterns = ['_build']
 
-# The reST default role (used for this markup: `text`) to use for all
+# The reST default role (used for this markup: `text`) to use for all documents.
-# documents.
 #default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
@@ -99,15 +98,12 @@ pygments_style = 'sphinx'
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []
 
-# If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
 
-
+# -- Options for HTML output ---------------------------------------------------
-# -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = 'alabaster'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -138,11 +134,6 @@ html_theme = 'default'
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#html_extra_path = []
-
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 #html_last_updated_fmt = '%b %d, %Y'
@@ -188,7 +179,7 @@ html_static_path = ['_static']
 htmlhelp_basename = 'gvawebdoc'
 
 
-# -- Options for LaTeX output ---------------------------------------------
+# -- Options for LaTeX output --------------------------------------------------
 
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
@@ -202,8 +193,7 @@ latex_elements = {
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
+# (source start file, target name, title, author, documentclass [howto/manual]).
-#  author, documentclass [howto, manual, or own class]).
 latex_documents = [
   ('index', 'gvaweb.tex', u'gvaweb Documentation',
    u'Jan Dittberner', 'manual'),
@@ -230,7 +220,7 @@ latex_documents = [
 #latex_domain_indices = True
 
 
-# -- Options for manual page output ---------------------------------------
+# -- Options for manual page output --------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
@@ -243,7 +233,7 @@ man_pages = [
 #man_show_urls = False
 
 
-# -- Options for Texinfo output -------------------------------------------
+# -- Options for Texinfo output ------------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
@@ -262,6 +252,3 @@ texinfo_documents = [
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False

docs/deploy.rst

@@ -7,4 +7,4 @@ of the following steps:
 * installation of native dependencies
 * setup of a virtualenv
 * installation of gvaweb production dependencies inside the virtualenv
-* setup of celery worker under control of supervisord
+* setup of celery worker under control of systemd

docs/install.rst

@@ -30,6 +30,7 @@ into the gvaweb directory and run the celery worker with:
 
 .. code-block:: sh
 
-    $ pipenv run celery -A gvaweb worker -Q web -l info
+    $ cd gvaweb
+    $ pipenv run celery -A webtasks worker -Q web -l info
 
 .. _Celery: http://www.celeryproject.org/

gvaweb.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+. /home/gvaweb/gvaweb-venv/bin/activate
+cd /srv/gvaweb/gvaweb
+celery -A webtasks worker -Q web -l info

gvaweb/gvaweb/webtasks/__init__.py

@@ -1,4 +0,0 @@
-"""
-This module contains :py:mod:`webtasks.tasks`.
-
-"""

gvaweb/webtasks/__init__.py

@@ -0,0 +1,10 @@
+"""
+This module contains :py:mod:`webtasks.tasks`.
+
+"""
+
+__version__ = "0.2.0"
+
+from webtasks.celery import app as celery_app
+
+__all__ = ("celery_app",)

gvaweb/webtasks/celery.py

@@ -7,7 +7,7 @@ This module defines the Celery_ app for gvaweb.
 from celery import Celery
 
 #: The Celery application
-app = Celery('gvaweb')
+app = Celery("webtasks")
 
-app.config_from_object('gvaweb.settings')
+app.config_from_object("webtasks.settings", namespace="CELERY")
-app.autodiscover_tasks(['gvaweb.webtasks'], force=True)
+app.autodiscover_tasks(["webtasks.tasks"], force=True)

gvaweb/webtasks/settings.py

@@ -25,22 +25,20 @@ def get_env_setting(setting):
 
 
 ########## CELERY CONFIGURATION
-CELERY_TIMEZONE = 'Europe/Berlin'
+CELERY_TIMEZONE = "Europe/Berlin"
 CELERY_ENABLE_UTC = True
-CELERY_RESULT_BACKEND = 'amqp'
 CELERY_RESULT_PERSISTENT = True
 CELERY_TASK_RESULT_EXPIRES = None
-CELERY_ROUTES = (
+CELERY_ROUTES = ("gvacommon.celeryrouters.GvaRouter",)
-    'gvacommon.celeryrouters.GvaRouter',
+CELERY_ACCEPT_CONTENT = ["json"]
-)
+CELERY_TASK_SERIALIZER = "json"
-CELERY_ACCEPT_CONTENT = ['json']
+CELERY_RESULT_SERIALIZER = "json"
-CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_BACKEND = get_env_setting("GVAWEB_RESULTS_REDIS_URL")
-CELERY_RESULT_SERIALIZER = 'json'
+CELERY_BROKER_URL = get_env_setting("GVAWEB_BROKER_URL")
-BROKER_URL = get_env_setting('GVAWEB_BROKER_URL')
 ########## END CELERY CONFIGURATION
 
 ########## GVAWEB CONFIGURATION
-GVAWEB_NGINX_SITES_AVAILABLE = get_env_setting('GVAWEB_NGINX_SITES_AVAILABLE')
+GVAWEB_NGINX_SITES_AVAILABLE = get_env_setting("GVAWEB_NGINX_SITES_AVAILABLE")
-GVAWEB_NGINX_SITES_ENABLED = get_env_setting('GVAWEB_NGINX_SITES_ENABLED')
+GVAWEB_NGINX_SITES_ENABLED = get_env_setting("GVAWEB_NGINX_SITES_ENABLED")
-GVAWEB_WWWUSER_MOUNT = get_env_setting('GVAWEB_WWWUSER_MOUNT')
+GVAWEB_WWWUSER_MOUNT = get_env_setting("GVAWEB_WWWUSER_MOUNT")
 ########## END GVAWEB CONFIGURATION

gvaweb/webtasks/tasks.py

@@ -10,9 +10,7 @@ from jinja2 import Environment, PackageLoader
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
-
+from webtasks import settings
-from gvaweb import settings
-
 
 _LOGGER = get_task_logger(__name__)
 
@@ -22,14 +20,14 @@ LN_CMD = '/bin/ln'
 SERVICE_CMD = '/bin/systemctl'
 INSTALL_CMD = '/usr/bin/install'
 
-JINJAENV = Environment(loader=PackageLoader('gvaweb.webtasks', 'templates'))
+JINJA_ENV = Environment(loader=PackageLoader('webtasks', 'templates'))
 
 
 def _jinja_parentdomain(domain):
     return '.'.join(domain.split('.')[1:])
 
 
-JINJAENV.filters['parentdomain'] = _jinja_parentdomain
+JINJA_ENV.filters['parentdomain'] = _jinja_parentdomain
 
 
 def log_and_raise(exception, message, *args):
@@ -57,7 +55,7 @@ def _build_document_root_path(sitename, username):
 
 
 def _get_template(templatename):
-    return JINJAENV.get_template(templatename)
+    return JINJA_ENV.get_template(templatename)
 
 
 @shared_task
@@ -74,16 +72,16 @@ def create_web_vhost_config(username, sitename, wildcard):
     :rtype: boolean
 
     """
-    conftmpl = _get_template('vhost.nginx')
+    config_template = _get_template('vhost.nginx')
-    confdata = conftmpl.render(domain=sitename, wildcard=wildcard)
+    config_data = config_template.render(domain=sitename, wildcard=wildcard)
-    conffile = None
+    config_file = None
     try:
-        nginxtemp, filename = mkstemp()
+        nginx_temp_file, filename = mkstemp()
-        conffile = os.fdopen(nginxtemp, 'w')
+        config_file = os.fdopen(nginx_temp_file, 'w')
-        conffile.write(confdata.encode('utf8'))
+        config_file.write(config_data.encode('utf8'))
     finally:
-        if conffile:
+        if config_file:
-            conffile.close()
+            config_file.close()
     try:
         subprocess.check_output([
             SUDO_CMD, INSTALL_CMD, '-o', 'root', '-g', 'root', '-m', '0640',
@@ -176,7 +174,7 @@ def create_web_php_fpm_pool_config(username):
     :rtype: boolean
 
     """
-    # TODO: implement PHP FPM docker management
+    # TODO: implement PHP FPM docker management setup
     return True
 
 
@@ -190,4 +188,5 @@ def delete_web_php_fpm_pool_config(username):
     :rtype: boolean
 
     """
+    # TODO: implement PHP FPM docker management removal
     return True