From 9f8ee2fa047b1ac4ca321555b9f2f8710ee4c0b4 Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Tue, 3 Mar 2020 13:13:26 +0100 Subject: [PATCH 1/5] Add Docker setup Add a Dockerfile and an entrypoint shell script. --- Dockerfile | 45 +++++++++++++++++++++++++++++++++++++++++++++ gvaweb.sh | 7 +++++++ 2 files changed, 52 insertions(+) create mode 100644 Dockerfile create mode 100755 gvaweb.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..03babe4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +ARG DEBIAN_RELEASE=buster +FROM debian:$DEBIAN_RELEASE +LABEL maintainer="Jan Dittberner " + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + dumb-init \ + gettext \ + git \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-virtualenv \ + python3-wheel \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*.* + +RUN python3 -m pip install --prefix=/usr/local pipenv + +ARG GVAGID=2000 +ARG GVAUID=2000 + +ARG GVAAPP=gvaweb + +WORKDIR /srv/$GVAAPP + +COPY Pipfile Pipfile.lock /srv/$GVAAPP/ + +RUN addgroup --gid $GVAGID $GVAAPP ; \ + adduser --home /home/$GVAAPP --shell /bin/bash --uid $GVAUID --gid $GVAGID --disabled-password --gecos "User for gnuviechadmin component $GVAAPP" $GVAAPP + +USER $GVAAPP +RUN python3 -m virtualenv --python=python3 /home/$GVAAPP/$GVAAPP-venv ; \ + /home/$GVAAPP/$GVAAPP-venv/bin/python3 -m pip install -U pip ; \ + VIRTUAL_ENV=/home/$GVAAPP/$GVAAPP-venv pipenv install --deploy --ignore-pipfile --dev + +VOLUME /srv/$GVAAPP + +COPY gvaweb.sh /srv/ + +ENTRYPOINT ["dumb-init", "/srv/gvaweb.sh"] diff --git a/gvaweb.sh b/gvaweb.sh new file mode 100755 index 0000000..e08a334 --- /dev/null +++ b/gvaweb.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +. /home/gvaweb/gvaweb-venv/bin/activate +cd /srv/gvaweb/gvaweb +celery -A gvaweb worker -Q web -l info From 9e48e068e23e142a1ff62fc4d94caf5bf04c9c98 Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Tue, 3 Mar 2020 15:22:26 +0100 Subject: [PATCH 2/5] Add redis dependency --- Pipfile | 1 + Pipfile.lock | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Pipfile b/Pipfile index b0f65c6..08f23f9 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ celery = "*" kombu = "*" pytz = "*" jinja2 = "*" +redis = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 5bb48e7..4f793a2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4420e939c194bdd75c095801149c37d0eb28c9e3010e20e701ee8d95e81b0a52" + "sha256": "19baf5d6f093d84ef837123087771dcd28d324a4b12e901f520374272ebbee75" }, "pipfile-spec": 6, "requires": { @@ -116,6 +116,14 @@ "index": "pypi", "version": "==2019.3" }, + "redis": { + "hashes": [ + "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f", + "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833" + ], + "index": "pypi", + "version": "==3.4.1" + }, "vine": { "hashes": [ "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87", @@ -125,10 +133,10 @@ }, "zipp": { "hashes": [ - "sha256:12248a63bbdf7548f89cb4c7cda4681e537031eda29c02ea29674bc6854460c2", - "sha256:7c0f8e91abc0dc07a5068f315c52cb30c66bfbc581e5b50704c8a2f6ebae794a" + "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", + "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96" ], - "version": "==3.0.0" + "version": "==3.1.0" } }, "develop": { From 3b05272254fc4ac1ad4a9f6f7ea6f700c2845ceb Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Tue, 3 Mar 2020 15:22:51 +0100 Subject: [PATCH 3/5] Move code from gvaweb to webtasks This change allows celery to find its tasks under the expected webtasks namespace. --- gvaweb.sh | 2 +- gvaweb/gvaweb/webtasks/__init__.py | 4 --- gvaweb/webtasks/__init__.py | 10 +++++++ .../__init__.py => webtasks/celery.py} | 6 ++-- gvaweb/{gvaweb => webtasks}/settings.py | 22 +++++++------- gvaweb/{gvaweb => }/webtasks/tasks.py | 29 +++++++++---------- .../webtasks/templates/vhost.nginx | 0 7 files changed, 38 insertions(+), 35 deletions(-) delete mode 100644 gvaweb/gvaweb/webtasks/__init__.py create mode 100644 gvaweb/webtasks/__init__.py rename gvaweb/{gvaweb/__init__.py => webtasks/celery.py} (51%) rename gvaweb/{gvaweb => webtasks}/settings.py (59%) rename gvaweb/{gvaweb => }/webtasks/tasks.py (87%) rename gvaweb/{gvaweb => }/webtasks/templates/vhost.nginx (100%) diff --git a/gvaweb.sh b/gvaweb.sh index e08a334..50cfe48 100755 --- a/gvaweb.sh +++ b/gvaweb.sh @@ -4,4 +4,4 @@ set -e . /home/gvaweb/gvaweb-venv/bin/activate cd /srv/gvaweb/gvaweb -celery -A gvaweb worker -Q web -l info +celery -A webtasks worker -Q web -l info diff --git a/gvaweb/gvaweb/webtasks/__init__.py b/gvaweb/gvaweb/webtasks/__init__.py deleted file mode 100644 index 9e02bbf..0000000 --- a/gvaweb/gvaweb/webtasks/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -This module contains :py:mod:`webtasks.tasks`. - -""" diff --git a/gvaweb/webtasks/__init__.py b/gvaweb/webtasks/__init__.py new file mode 100644 index 0000000..1ade301 --- /dev/null +++ b/gvaweb/webtasks/__init__.py @@ -0,0 +1,10 @@ +""" +This module contains :py:mod:`webtasks.tasks`. + +""" + +__version__ = "0.2.0" + +from webtasks.celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/gvaweb/gvaweb/__init__.py b/gvaweb/webtasks/celery.py similarity index 51% rename from gvaweb/gvaweb/__init__.py rename to gvaweb/webtasks/celery.py index ebf6b6f..a38923f 100644 --- a/gvaweb/gvaweb/__init__.py +++ b/gvaweb/webtasks/celery.py @@ -7,7 +7,7 @@ This module defines the Celery_ app for gvaweb. from celery import Celery #: The Celery application -app = Celery('gvaweb') +app = Celery("webtasks") -app.config_from_object('gvaweb.settings') -app.autodiscover_tasks(['gvaweb.webtasks'], force=True) +app.config_from_object("webtasks.settings", namespace="CELERY") +app.autodiscover_tasks(["webtasks.tasks"], force=True) diff --git a/gvaweb/gvaweb/settings.py b/gvaweb/webtasks/settings.py similarity index 59% rename from gvaweb/gvaweb/settings.py rename to gvaweb/webtasks/settings.py index f31cf6c..1398dc8 100644 --- a/gvaweb/gvaweb/settings.py +++ b/gvaweb/webtasks/settings.py @@ -25,22 +25,20 @@ def get_env_setting(setting): ########## CELERY CONFIGURATION -CELERY_TIMEZONE = 'Europe/Berlin' +CELERY_TIMEZONE = "Europe/Berlin" CELERY_ENABLE_UTC = True -CELERY_RESULT_BACKEND = 'amqp' CELERY_RESULT_PERSISTENT = True CELERY_TASK_RESULT_EXPIRES = None -CELERY_ROUTES = ( - 'gvacommon.celeryrouters.GvaRouter', -) -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_RESULT_SERIALIZER = 'json' -BROKER_URL = get_env_setting('GVAWEB_BROKER_URL') +CELERY_ROUTES = ("gvacommon.celeryrouters.GvaRouter",) +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" +CELERY_RESULT_BACKEND = get_env_setting("GVAWEB_RESULTS_REDIS_URL") +CELERY_BROKER_URL = get_env_setting("GVAWEB_BROKER_URL") ########## END CELERY CONFIGURATION ########## GVAWEB CONFIGURATION -GVAWEB_NGINX_SITES_AVAILABLE = get_env_setting('GVAWEB_NGINX_SITES_AVAILABLE') -GVAWEB_NGINX_SITES_ENABLED = get_env_setting('GVAWEB_NGINX_SITES_ENABLED') -GVAWEB_WWWUSER_MOUNT = get_env_setting('GVAWEB_WWWUSER_MOUNT') +GVAWEB_NGINX_SITES_AVAILABLE = get_env_setting("GVAWEB_NGINX_SITES_AVAILABLE") +GVAWEB_NGINX_SITES_ENABLED = get_env_setting("GVAWEB_NGINX_SITES_ENABLED") +GVAWEB_WWWUSER_MOUNT = get_env_setting("GVAWEB_WWWUSER_MOUNT") ########## END GVAWEB CONFIGURATION diff --git a/gvaweb/gvaweb/webtasks/tasks.py b/gvaweb/webtasks/tasks.py similarity index 87% rename from gvaweb/gvaweb/webtasks/tasks.py rename to gvaweb/webtasks/tasks.py index e36940a..e102f57 100644 --- a/gvaweb/gvaweb/webtasks/tasks.py +++ b/gvaweb/webtasks/tasks.py @@ -10,9 +10,7 @@ from jinja2 import Environment, PackageLoader from celery import shared_task from celery.utils.log import get_task_logger - -from gvaweb import settings - +from webtasks import settings _LOGGER = get_task_logger(__name__) @@ -22,14 +20,14 @@ LN_CMD = '/bin/ln' SERVICE_CMD = '/bin/systemctl' INSTALL_CMD = '/usr/bin/install' -JINJAENV = Environment(loader=PackageLoader('gvaweb.webtasks', 'templates')) +JINJA_ENV = Environment(loader=PackageLoader('webtasks', 'templates')) def _jinja_parentdomain(domain): return '.'.join(domain.split('.')[1:]) -JINJAENV.filters['parentdomain'] = _jinja_parentdomain +JINJA_ENV.filters['parentdomain'] = _jinja_parentdomain def log_and_raise(exception, message, *args): @@ -57,7 +55,7 @@ def _build_document_root_path(sitename, username): def _get_template(templatename): - return JINJAENV.get_template(templatename) + return JINJA_ENV.get_template(templatename) @shared_task @@ -74,16 +72,16 @@ def create_web_vhost_config(username, sitename, wildcard): :rtype: boolean """ - conftmpl = _get_template('vhost.nginx') - confdata = conftmpl.render(domain=sitename, wildcard=wildcard) - conffile = None + config_template = _get_template('vhost.nginx') + config_data = config_template.render(domain=sitename, wildcard=wildcard) + config_file = None try: - nginxtemp, filename = mkstemp() - conffile = os.fdopen(nginxtemp, 'w') - conffile.write(confdata.encode('utf8')) + nginx_temp_file, filename = mkstemp() + config_file = os.fdopen(nginx_temp_file, 'w') + config_file.write(config_data.encode('utf8')) finally: - if conffile: - conffile.close() + if config_file: + config_file.close() try: subprocess.check_output([ SUDO_CMD, INSTALL_CMD, '-o', 'root', '-g', 'root', '-m', '0640', @@ -176,7 +174,7 @@ def create_web_php_fpm_pool_config(username): :rtype: boolean """ - # TODO: implement PHP FPM docker management + # TODO: implement PHP FPM docker management setup return True @@ -190,4 +188,5 @@ def delete_web_php_fpm_pool_config(username): :rtype: boolean """ + # TODO: implement PHP FPM docker management removal return True diff --git a/gvaweb/gvaweb/webtasks/templates/vhost.nginx b/gvaweb/webtasks/templates/vhost.nginx similarity index 100% rename from gvaweb/gvaweb/webtasks/templates/vhost.nginx rename to gvaweb/webtasks/templates/vhost.nginx From 07dee3430510d4864e6f8497f31458f756052b14 Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Tue, 3 Mar 2020 15:24:11 +0100 Subject: [PATCH 4/5] Update documentation - use alabaster theme - update changelog - use version number from code module --- docs/changelog.rst | 5 +++++ docs/code.rst | 23 ++++++++------------- docs/conf.py | 51 +++++++++++++++++----------------------------- docs/deploy.rst | 2 +- docs/install.rst | 3 ++- 5 files changed, 36 insertions(+), 48 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 29e6b9a..3265a4a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,11 @@ Changelog ========= +* :support:`-` add Docker setup for lightweight local testing +* :support:`-` restructure code to have celery in the webtasks module and drop + the gvaldap module +* :support:`-` update documentation to alabaster theme + * :release:`0.1.8 <2020-03-02>` * :bug:`-` salt automation was broken due to changed paths and missing Jinja 2 dependency diff --git a/docs/code.rst b/docs/code.rst index a379324..eaa03ef 100644 --- a/docs/code.rst +++ b/docs/code.rst @@ -7,30 +7,25 @@ gvaweb is implemented as `Celery`_ app. .. _Celery: http://www.celeryproject.org/ -The project module :py:mod:`gvaweb` -=================================== +The module :py:mod:`webtasks` +============================= -.. automodule:: gvaweb +.. automodule:: webtasks -:py:mod:`celery ` --------------------------------- +:py:mod:`celery ` +---------------------------------- -.. automodule:: gvaweb.celery +.. automodule:: webtasks.celery :members: -:py:mod:`settings ` ------------------------------------- +:py:mod:`settings ` +-------------------------------------- -.. automodule:: gvaweb.settings +.. automodule:: webtasks.settings -:py:mod:`webtasks` app -====================== - -.. automodule:: webtasks - :py:mod:`tasks ` -------------------------------- diff --git a/docs/conf.py b/docs/conf.py index 9457666..72c23e8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- +# pymode:lint_ignore=E501 # # gvaweb documentation build configuration file, created by # sphinx-quickstart on Mon Jan 26 15:51:21 2015. # -# This file is execfile()d with the current directory set to its -# containing dir. +# This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. @@ -30,18 +30,17 @@ os.environ['GVAWEB_WWWUSER_MOUNT'] = '/srv/wwwfiles' # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['releases', 'sphinx.ext.autodoc', 'celery.contrib.sphinx'] +# configuration for releases extension +releases_issue_uri = 'https://git.dittberner.info/gnuviech/gvaweb/issues/%s' +releases_release_uri = 'https://git.dittberner.info/gnuviech/gvaweb/src/tag/%s' + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -releases_issue_uri = 'https://git.dittberner.info/gnuviech/gvaweb/issues/%s' - -releases_release_uri = 'https://git.dittberner.info/gnuviech/gvaweb/src/tag/%s' - # The suffix of source filenames. source_suffix = '.rst' @@ -59,10 +58,11 @@ copyright = u'2015-2020, Jan Dittberner' # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = '0.1' # The full version, including alpha/beta/rc tags. -release = '0.1.8' +from webtasks import __version__ as release + +# The short X.Y version. +version = ".".join(release.split('.')[:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -78,8 +78,7 @@ release = '0.1.8' # directories to ignore when looking for source files. exclude_patterns = ['_build'] -# The reST default role (used for this markup: `text`) to use for all -# documents. +# The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. @@ -99,15 +98,12 @@ pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - -# -- Options for HTML output ---------------------------------------------- +# -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -138,11 +134,6 @@ html_theme = 'default' # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' @@ -188,7 +179,7 @@ html_static_path = ['_static'] htmlhelp_basename = 'gvawebdoc' -# -- Options for LaTeX output --------------------------------------------- +# -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). @@ -202,8 +193,7 @@ latex_elements = { } # Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). +# (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'gvaweb.tex', u'gvaweb Documentation', u'Jan Dittberner', 'manual'), @@ -230,7 +220,7 @@ latex_documents = [ #latex_domain_indices = True -# -- Options for manual page output --------------------------------------- +# -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). @@ -243,7 +233,7 @@ man_pages = [ #man_show_urls = False -# -- Options for Texinfo output ------------------------------------------- +# -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, @@ -262,6 +252,3 @@ texinfo_documents = [ # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False diff --git a/docs/deploy.rst b/docs/deploy.rst index 11ceb7e..1c70387 100644 --- a/docs/deploy.rst +++ b/docs/deploy.rst @@ -7,4 +7,4 @@ of the following steps: * installation of native dependencies * setup of a virtualenv * installation of gvaweb production dependencies inside the virtualenv -* setup of celery worker under control of supervisord +* setup of celery worker under control of systemd diff --git a/docs/install.rst b/docs/install.rst index b7d9eec..2501a69 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -30,6 +30,7 @@ into the gvaweb directory and run the celery worker with: .. code-block:: sh - $ pipenv run celery -A gvaweb worker -Q web -l info + $ cd gvaweb + $ pipenv run celery -A webtasks worker -Q web -l info .. _Celery: http://www.celeryproject.org/ From 782128836b139aa89299ae719508c1173b360cf1 Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Tue, 3 Mar 2020 15:26:43 +0100 Subject: [PATCH 5/5] Add release version to changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3265a4a..b2a6793 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,7 @@ Changelog ========= +* :release:`0.2.0 <2020-03-03>` * :support:`-` add Docker setup for lightweight local testing * :support:`-` restructure code to have celery in the webtasks module and drop the gvaldap module