Make gva run with Poetry and logging

pillar/gnuviechadmin/gva.sls

@@ -9,9 +9,9 @@ gnuviechadmin:
   database:
     host: pgsql
   gva:
-    django_secret_key: yBnbG4azhNaTxIW0/Rv2dEij9PcVU1KVR//1bR6LujmLBnZJw8OOrEi2dIqz3pyOdG8=
+    django_secret_key: "yBnbG4azhNaTxIW0/Rv2dEij9PcVU1KVR//1bR6LujmLBnZJw8OOrEi2dIqz3pyOdG8="
     fullname: Self Service Web Interface
-    git_branch: master
+    git_branch: main
     git_url: https://git.dittberner.info/gnuviech/gva.git
     url_mysql_admin: https://phpmyadmin.gva.local/
     url_pgsql_admin: https://phppgadmin.gva.local/

states/gnuviechadmin/gva.sls

@@ -1,8 +1,8 @@
+---
 {% set gvaappname = salt['pillar.get']('gnuviechadmin:appname') %}
 {% set app_home = salt['grains.get']('gnuviechadmin:home', '/home/{}'.format(gvaappname)) %}
 {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %}
 {% set app_group = salt['grains.get']('gnuviechadmin:group', gvaappname) %}
-{% set venv = "{}/{}-venv".format(app_home, gvaappname) -%}
 
 {% set amqp_user = salt['pillar.get']('gnuviechadmin:{}:amqp_user'.format(gvaappname), gvaappname) -%}
 {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%}
@@ -13,8 +13,7 @@
 {% from 'gnuviechadmin/gvaapp_macros.sls' import gvaapp_base with context %}
 include:
 - base
-- python.pipenv
-- python.virtualenv
+- python.poetry
 - uwsgi.python3
 
 {{ gvaapp_base(gvaappname, 'uwsgi') }}
@@ -25,39 +24,34 @@ include:
       - libpq-dev
     - require_in:
       - cmd: {{ gvaappname }}-requirements
+      - service: uwsgi
 
 gettext:
   pkg.installed
 
-{{ checkout }}/.env:
+{{ checkout }}/run{{ gvaappname }}:
   file.managed:
     - user: {{ app_user }}
     - group: {{ app_group }}
-    - mode: 0640
-    - source: salt://gnuviechadmin/{{ gvaappname }}/env-vars
+    - mode: 0750
+    - source: salt://gnuviechadmin/{{ gvaappname }}/run.sh
     - template: jinja
     - context:
-        gvaappname: {{ gvaappname }}
         broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }}
         result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0
+        gvaappname: {{ gvaappname }}
+        checkout: {{ checkout }}
     - require:
-      - user: {{ gvaappname }}-user
-      - group: {{ gvaappname }}-group
-      - file: {{ checkout }}
+      - git: {{ gitrepo }}
 
 {% for command in ['migrate --noinput', 'collectstatic --noinput', 'compilemessages'] %}
 {{ gvaappname }}-manage-{{ command }}:
   cmd.wait:
-    - name: /usr/local/bin/pipenv run python3 manage.py {{ command }}
+    - name: {{ checkout }}/run{{ gvaappname }} {{ command }}
     - runas: {{ app_user }}
     - cwd: {{ checkout }}/gnuviechadmin
-    - env:
-      - VIRTUAL_ENV: "{{ venv }}"
-      - LC_ALL: C.UTF-8
-      - LANG: C.UTF-8
     - watch:
       - cmd: {{ gvaappname }}-requirements
-      - file: {{ checkout }}/.env
       {%- if update_git %}
       - git: {{ gitrepo }}
       {%- endif %}
@@ -75,8 +69,9 @@ gettext:
         broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }}
         result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0
         workdir: {{ checkout }}/gnuviechadmin
-        venv: {{ venv }}
+        venv: {{ checkout }}/.venv
     - require:
+      - group: {{ gvaappname }}-group
       - pkg: uwsgi
     - require_in:
       - service: uwsgi

states/gnuviechadmin/gva/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -e
+cd "{{ checkout }}/gnuviechadmin"
+
+export DJANGO_SETTINGS_MODULE=gnuviechadmin.settings
+export GVA_ADMIN_EMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }}
+export GVA_ADMIN_NAME={{ salt['pillar.get']('gnuviechadmin:adminname', 'Gnuviech Admin') }}
+export GVA_BROKER_URL={{ broker_url }}
+export GVA_DOMAIN_NAME={{ salt['pillar.get']('gnuviechadmin:{}:domainname'.format(gvaappname), 'service.localhost') }}
+export GVA_LOG_FILE=/var/log/gnuviechadmin/{{ gvaappname }}.log
+export GVA_MIN_OS_GID={{ salt['pillar.get']('gnuviechadmin:minosgid', 10000) }}
+export GVA_MIN_OS_UID={{ salt['pillar.get']('gnuviechadmin:minosuid', 10000) }}
+export GVA_OSUSER_DEFAULT_SHELL={{ salt['pillar.get']('gnuviechadmin:osuserdefaultshell', '/sbin/nologin') }}
+export GVA_OSUSER_HOME_BASEPATH={{ salt['pillar.get']('gnuviechadmin:osuserhomedirbase', '/home') }}
+export GVA_OSUSER_PREFIX={{ salt['pillar.get']('gnuviechadmin:osuserprefix', 'user') }}
+export GVA_OSUSER_UPLOADSERVER={{ salt['pillar.get']('gnuviechadmin:uploadserver') }}
+export GVA_PGSQL_DATABASE={{ salt['pillar.get']('gnuviechadmin:database:name') }}
+export GVA_PGSQL_HOSTNAME={{ salt['pillar.get']('gnuviechadmin:database:host', 'localhost') }}
+export GVA_PGSQL_PASSWORD={{ salt['pillar.get']('gnuviechadmin:database:owner:password') }}
+export GVA_PGSQL_PORT={{ salt['pillar.get']('gnuviechadmin:database:port', 5432) }}
+export GVA_PGSQL_USER={{ salt['pillar.get']('gnuviechadmin:database:owner:user', gvaappname ) }}
+export GVA_RESULTS_REDIS_URL={{ result_url }}
+export GVA_SITE_ADMINMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }}
+export GVA_SITE_NAME={{ salt['pillar.get']('gnuviechadmin:sitename') }}
+export GVA_SITE_SECRET={{ salt['pillar.get']('gnuviechadmin:{}:django_secret_key'.format(gvaappname)) }}
+export GVA_URL_MYSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_mysql_admin'.format(gvaappname)) }}
+export GVA_URL_PGSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_pgsql_admin'.format(gvaappname)) }}
+export GVA_URL_WEBMAIL={{ salt['pillar.get']('gnuviechadmin:{}:url_webmail'.format(gvaappname)) }}
+export LANG=C.UTF-8
+export LC_ALL=C.UTF-8
+
+{{ checkout }}/.venv/bin/python3 manage.py $*

states/gnuviechadmin/gva/uwsgi.ini

@@ -15,6 +15,7 @@ env = GVA_ADMIN_EMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@e
 env = GVA_ADMIN_NAME={{ salt['pillar.get']('gnuviechadmin:adminname', 'Gnuviech Admin') }}
 env = GVA_BROKER_URL={{ broker_url }}
 env = GVA_DOMAIN_NAME={{ salt['pillar.get']('gnuviechadmin:{}:domainname'.format(gvaappname), 'service.localhost') }}
+env = GVA_LOG_FILE=/var/log/gnuviechadmin/{{ gvaappname }}.log
 env = GVA_MIN_OS_GID={{ salt['pillar.get']('gnuviechadmin:minosgid', 10000) }}
 env = GVA_MIN_OS_UID={{ salt['pillar.get']('gnuviechadmin:minosuid', 10000) }}
 env = GVA_OSUSER_DEFAULT_SHELL={{ salt['pillar.get']('gnuviechadmin:osuserdefaultshell', '/sbin/nologin') }}

states/gnuviechadmin/gvaapp_macros.sls

@@ -2,12 +2,12 @@
 {% set app_home = salt['grains.get']('gnuviechadmin:home', '/home/{}'.format(gvaappname)) %}
 {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %}
 {% set app_group = salt['grains.get']('gnuviechadmin:group', gvaappname) %}
-{% set venv = "{}/{}-venv".format(app_home, gvaappname) -%}
 
 {% set appfullname = 'GNUViech Admin {} User'.format(salt['pillar.get']('gnuviechadmin:{}:fullname'.format(gvaappname))) -%}
 {% set update_git = salt['grains.get']('gnuviechadmin:update_git', True) %}
 {% set gitrepo = salt['pillar.get']('gnuviechadmin:{}:git_url'.format(gvaappname), 'git:gnuviech/{}.git'.format(gvaappname)) -%}
 {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%}
+{% set venv = "{}/.venv".format(checkout) -%}
 {% set deployment_key = '{}/.ssh/id_deployment'.format(app_home) -%}
 
 {% for host in salt['pillar.get']('gnuviechadmin:machines', {}) %}
@@ -45,52 +45,14 @@ gvabase-dependencies:
   pkg.installed:
     - name: build-essential
 
-{% if update_git %}
-{{ app_home }}/.ssh:
+/var/log/gnuviechadmin:
   file.directory:
     - user: {{ app_user }}
     - group: {{ app_group }}
-    - mode: 0700
+    - mode: 0750
     - require:
       - user: {{ gvaappname }}-user
-
-SSH Deployment Key:
-  cmd.run:
-    - name: ssh-keygen -t ed25519 -C "Deployment key for {{ gvaappname }}" -N "" -f {{ deployment_key }}
-    - creates: {{ deployment_key }}
-    - runas: {{ app_user }}
-    - requires:
-      - file: {{ app_home }}/.ssh
-    - require_in:
-      - git: {{ gitrepo }}
-
-SSH known hosts configuration:
-  file.managed:
-    - name: {{ app_home }}/.ssh/known_hosts
-    - user: {{ app_user }}
-    - group: {{ app_group }}
-    - mode: 0600
-    - contents_pillar: gnuviechadmin:ssh_known_hosts
-    - require:
-      - file: {{ app_home }}/.ssh
-    - require_in:
-      - git: {{ gitrepo }}
-
-SSH configuration:
-  file.managed:
-    - name: {{ app_home }}/.ssh/config
-    - user: {{ app_user }}
-    - group: {{ app_group }}
-    - mode: 0600
-    - source: salt://gnuviechadmin/ssh_deploy_config
-    - template: jinja
-    - context:
-        key: {{ deployment_key }}
-    - require:
-      - file: {{ app_home }}/.ssh
-    - require_in:
-      - git: {{ gitrepo }}
-{% endif %}
+      - group: {{ gvaappname }}-group
 
 {{ checkout }}:
   file.directory:
@@ -106,6 +68,7 @@ SSH configuration:
     - user: {{ app_user }}
     - target: {{ checkout }}
     - rev: {{ salt['pillar.get']('gnuviechadmin:{}:git_branch'.format(gvaappname), 'production') }}
+    - force_clone: true
     - require:
       - file: {{ checkout }}
     - watch_in:
@@ -113,55 +76,17 @@ SSH configuration:
       - service: {{ servicename }}
 {% endif %}
 
-rm -rf {{ venv }}:
-  cmd.run:
-    - runas: {{ app_user }}
-    - unless: test -f {{ venv }}/bin/python3
-    - require:
-      - user: {{ gvaappname }}-user
-
-create-{{ gvaappname }}-venv:
-  cmd.run:
-    - name: python3 -m virtualenv --python=python3 {{ venv }}
-    - runas: {{ app_user }}
-    - unless: test -f {{ venv }}/bin/pip3
-    - require:
-      - user: {{ gvaappname }}-user
-      - python3-virtualenv-packages
-    - watch_in:
-      - cmd: update-{{ gvaappname }}-pip
-
-update-{{ gvaappname }}-pip:
-  cmd.wait:
-    - name: {{ venv }}/bin/python3 -m pip install -U pip
-    - runas: {{ app_user }}
-    - require:
-      - user: {{ gvaappname }}-user
-
-{{ venv }}:
-  file.directory:
-    - user: {{ app_user }}
-    - group: {{ app_group }}
-    - require:
-      - cmd: create-{{ gvaappname }}-venv
-    - watch_in:
-      - cmd: {{ gvaappname }}-requirements
-
 {{ gvaappname }}-requirements:
   cmd.wait:
-    - name: /usr/local/bin/pipenv install --deploy
+    - name: /usr/local/poetry/bin/poetry install
     - runas: {{ app_user }}
     - cwd: {{ checkout }}
     - env:
-      - VIRTUAL_ENV: "{{ venv }}"
-      - PIPENV_HIDE_EMOJIS: 1
-      - PIPENV_NOSPIN: 1
-      - PIPENV_COLORBLIND: 1
+      - POETRY_VIRTUALENVS_IN_PROJECT: "true"
       - LC_ALL: C.UTF-8
       - LANG: C.UTF-8
     - require:
-      - cmd: install_pipenv
-      - file: {{ venv }}
+      - cmd: install_poetry
       {%- if update_git %}
       - git: {{ gitrepo }}
       {%- else %}

states/python/poetry.sls

@@ -0,0 +1,22 @@
+---
+curl:
+  pkg.installed
+
+python3:
+  pkg.installed
+
+poetry-preconditions:
+  pkg.installed:
+    - pkgs:
+      - python3-wheel
+      - python3-pip
+      - python3-setuptools
+
+install_poetry:
+  cmd.run:
+    - name: curl -sSL https://install.python-poetry.org | POETRY_VERSION={{ salt['grains.get']('python:poetry:version', '1.4.2') }} POETRY_HOME=/usr/local/poetry /usr/bin/python3 -
+    - creates: /usr/local/poetry/bin/poetry
+    - require:
+      - id: poetry-preconditions
+      - pkg: curl
+      - pkg: python3

states/uwsgi/init.sls

@@ -1,8 +1,9 @@
+---
 uwsgi:
   pkg:
     - installed
   service.running:
-    - enable: True
-    - reload: True
+    - enable: true
+    - reload: false
     - require:
       - pkg: uwsgi