From 18f0061ee4ac053fe0673edcc36d7f8c2e6bc339 Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Sat, 6 May 2023 14:42:50 +0200 Subject: [PATCH] Make gva run with Poetry and logging --- pillar/gnuviechadmin/gva.sls | 4 +- states/gnuviechadmin/gva.sls | 29 ++++---- states/gnuviechadmin/gva/run.sh | 33 ++++++++++ states/gnuviechadmin/gva/uwsgi.ini | 1 + states/gnuviechadmin/gvaapp_macros.sls | 91 +++----------------------- states/python/poetry.sls | 22 +++++++ states/uwsgi/init.sls | 5 +- 7 files changed, 81 insertions(+), 104 deletions(-) create mode 100644 states/gnuviechadmin/gva/run.sh create mode 100644 states/python/poetry.sls diff --git a/pillar/gnuviechadmin/gva.sls b/pillar/gnuviechadmin/gva.sls index def269a..adacf1f 100644 --- a/pillar/gnuviechadmin/gva.sls +++ b/pillar/gnuviechadmin/gva.sls @@ -9,9 +9,9 @@ gnuviechadmin: database: host: pgsql gva: - django_secret_key: yBnbG4azhNaTxIW0/Rv2dEij9PcVU1KVR//1bR6LujmLBnZJw8OOrEi2dIqz3pyOdG8= + django_secret_key: "yBnbG4azhNaTxIW0/Rv2dEij9PcVU1KVR//1bR6LujmLBnZJw8OOrEi2dIqz3pyOdG8=" fullname: Self Service Web Interface - git_branch: master + git_branch: main git_url: https://git.dittberner.info/gnuviech/gva.git url_mysql_admin: https://phpmyadmin.gva.local/ url_pgsql_admin: https://phppgadmin.gva.local/ diff --git a/states/gnuviechadmin/gva.sls b/states/gnuviechadmin/gva.sls index d35f873..0956974 100644 --- a/states/gnuviechadmin/gva.sls +++ b/states/gnuviechadmin/gva.sls @@ -1,8 +1,8 @@ +--- {% set gvaappname = salt['pillar.get']('gnuviechadmin:appname') %} {% set app_home = salt['grains.get']('gnuviechadmin:home', '/home/{}'.format(gvaappname)) %} {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %} {% set app_group = salt['grains.get']('gnuviechadmin:group', gvaappname) %} -{% set venv = "{}/{}-venv".format(app_home, gvaappname) -%} {% set amqp_user = salt['pillar.get']('gnuviechadmin:{}:amqp_user'.format(gvaappname), gvaappname) -%} {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%} @@ -13,8 +13,7 @@ {% from 'gnuviechadmin/gvaapp_macros.sls' import gvaapp_base with context %} include: - base -- python.pipenv -- python.virtualenv +- python.poetry - uwsgi.python3 {{ gvaapp_base(gvaappname, 'uwsgi') }} @@ -25,39 +24,34 @@ include: - libpq-dev - require_in: - cmd: {{ gvaappname }}-requirements + - service: uwsgi gettext: pkg.installed -{{ checkout }}/.env: +{{ checkout }}/run{{ gvaappname }}: file.managed: - user: {{ app_user }} - group: {{ app_group }} - - mode: 0640 - - source: salt://gnuviechadmin/{{ gvaappname }}/env-vars + - mode: 0750 + - source: salt://gnuviechadmin/{{ gvaappname }}/run.sh - template: jinja - context: - gvaappname: {{ gvaappname }} broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }} result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0 + gvaappname: {{ gvaappname }} + checkout: {{ checkout }} - require: - - user: {{ gvaappname }}-user - - group: {{ gvaappname }}-group - - file: {{ checkout }} + - git: {{ gitrepo }} {% for command in ['migrate --noinput', 'collectstatic --noinput', 'compilemessages'] %} {{ gvaappname }}-manage-{{ command }}: cmd.wait: - - name: /usr/local/bin/pipenv run python3 manage.py {{ command }} + - name: {{ checkout }}/run{{ gvaappname }} {{ command }} - runas: {{ app_user }} - cwd: {{ checkout }}/gnuviechadmin - - env: - - VIRTUAL_ENV: "{{ venv }}" - - LC_ALL: C.UTF-8 - - LANG: C.UTF-8 - watch: - cmd: {{ gvaappname }}-requirements - - file: {{ checkout }}/.env {%- if update_git %} - git: {{ gitrepo }} {%- endif %} @@ -75,8 +69,9 @@ gettext: broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }} result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0 workdir: {{ checkout }}/gnuviechadmin - venv: {{ venv }} + venv: {{ checkout }}/.venv - require: + - group: {{ gvaappname }}-group - pkg: uwsgi - require_in: - service: uwsgi diff --git a/states/gnuviechadmin/gva/run.sh b/states/gnuviechadmin/gva/run.sh new file mode 100644 index 0000000..c3277c9 --- /dev/null +++ b/states/gnuviechadmin/gva/run.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +set -e +cd "{{ checkout }}/gnuviechadmin" + +export DJANGO_SETTINGS_MODULE=gnuviechadmin.settings +export GVA_ADMIN_EMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }} +export GVA_ADMIN_NAME={{ salt['pillar.get']('gnuviechadmin:adminname', 'Gnuviech Admin') }} +export GVA_BROKER_URL={{ broker_url }} +export GVA_DOMAIN_NAME={{ salt['pillar.get']('gnuviechadmin:{}:domainname'.format(gvaappname), 'service.localhost') }} +export GVA_LOG_FILE=/var/log/gnuviechadmin/{{ gvaappname }}.log +export GVA_MIN_OS_GID={{ salt['pillar.get']('gnuviechadmin:minosgid', 10000) }} +export GVA_MIN_OS_UID={{ salt['pillar.get']('gnuviechadmin:minosuid', 10000) }} +export GVA_OSUSER_DEFAULT_SHELL={{ salt['pillar.get']('gnuviechadmin:osuserdefaultshell', '/sbin/nologin') }} +export GVA_OSUSER_HOME_BASEPATH={{ salt['pillar.get']('gnuviechadmin:osuserhomedirbase', '/home') }} +export GVA_OSUSER_PREFIX={{ salt['pillar.get']('gnuviechadmin:osuserprefix', 'user') }} +export GVA_OSUSER_UPLOADSERVER={{ salt['pillar.get']('gnuviechadmin:uploadserver') }} +export GVA_PGSQL_DATABASE={{ salt['pillar.get']('gnuviechadmin:database:name') }} +export GVA_PGSQL_HOSTNAME={{ salt['pillar.get']('gnuviechadmin:database:host', 'localhost') }} +export GVA_PGSQL_PASSWORD={{ salt['pillar.get']('gnuviechadmin:database:owner:password') }} +export GVA_PGSQL_PORT={{ salt['pillar.get']('gnuviechadmin:database:port', 5432) }} +export GVA_PGSQL_USER={{ salt['pillar.get']('gnuviechadmin:database:owner:user', gvaappname ) }} +export GVA_RESULTS_REDIS_URL={{ result_url }} +export GVA_SITE_ADMINMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }} +export GVA_SITE_NAME={{ salt['pillar.get']('gnuviechadmin:sitename') }} +export GVA_SITE_SECRET={{ salt['pillar.get']('gnuviechadmin:{}:django_secret_key'.format(gvaappname)) }} +export GVA_URL_MYSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_mysql_admin'.format(gvaappname)) }} +export GVA_URL_PGSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_pgsql_admin'.format(gvaappname)) }} +export GVA_URL_WEBMAIL={{ salt['pillar.get']('gnuviechadmin:{}:url_webmail'.format(gvaappname)) }} +export LANG=C.UTF-8 +export LC_ALL=C.UTF-8 + +{{ checkout }}/.venv/bin/python3 manage.py $* diff --git a/states/gnuviechadmin/gva/uwsgi.ini b/states/gnuviechadmin/gva/uwsgi.ini index 36e8447..dc3bebd 100644 --- a/states/gnuviechadmin/gva/uwsgi.ini +++ b/states/gnuviechadmin/gva/uwsgi.ini @@ -15,6 +15,7 @@ env = GVA_ADMIN_EMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@e env = GVA_ADMIN_NAME={{ salt['pillar.get']('gnuviechadmin:adminname', 'Gnuviech Admin') }} env = GVA_BROKER_URL={{ broker_url }} env = GVA_DOMAIN_NAME={{ salt['pillar.get']('gnuviechadmin:{}:domainname'.format(gvaappname), 'service.localhost') }} +env = GVA_LOG_FILE=/var/log/gnuviechadmin/{{ gvaappname }}.log env = GVA_MIN_OS_GID={{ salt['pillar.get']('gnuviechadmin:minosgid', 10000) }} env = GVA_MIN_OS_UID={{ salt['pillar.get']('gnuviechadmin:minosuid', 10000) }} env = GVA_OSUSER_DEFAULT_SHELL={{ salt['pillar.get']('gnuviechadmin:osuserdefaultshell', '/sbin/nologin') }} diff --git a/states/gnuviechadmin/gvaapp_macros.sls b/states/gnuviechadmin/gvaapp_macros.sls index 518a752..85b196f 100644 --- a/states/gnuviechadmin/gvaapp_macros.sls +++ b/states/gnuviechadmin/gvaapp_macros.sls @@ -2,12 +2,12 @@ {% set app_home = salt['grains.get']('gnuviechadmin:home', '/home/{}'.format(gvaappname)) %} {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %} {% set app_group = salt['grains.get']('gnuviechadmin:group', gvaappname) %} -{% set venv = "{}/{}-venv".format(app_home, gvaappname) -%} {% set appfullname = 'GNUViech Admin {} User'.format(salt['pillar.get']('gnuviechadmin:{}:fullname'.format(gvaappname))) -%} {% set update_git = salt['grains.get']('gnuviechadmin:update_git', True) %} {% set gitrepo = salt['pillar.get']('gnuviechadmin:{}:git_url'.format(gvaappname), 'git:gnuviech/{}.git'.format(gvaappname)) -%} {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%} +{% set venv = "{}/.venv".format(checkout) -%} {% set deployment_key = '{}/.ssh/id_deployment'.format(app_home) -%} {% for host in salt['pillar.get']('gnuviechadmin:machines', {}) %} @@ -45,52 +45,14 @@ gvabase-dependencies: pkg.installed: - name: build-essential -{% if update_git %} -{{ app_home }}/.ssh: +/var/log/gnuviechadmin: file.directory: - user: {{ app_user }} - group: {{ app_group }} - - mode: 0700 + - mode: 0750 - require: - user: {{ gvaappname }}-user - -SSH Deployment Key: - cmd.run: - - name: ssh-keygen -t ed25519 -C "Deployment key for {{ gvaappname }}" -N "" -f {{ deployment_key }} - - creates: {{ deployment_key }} - - runas: {{ app_user }} - - requires: - - file: {{ app_home }}/.ssh - - require_in: - - git: {{ gitrepo }} - -SSH known hosts configuration: - file.managed: - - name: {{ app_home }}/.ssh/known_hosts - - user: {{ app_user }} - - group: {{ app_group }} - - mode: 0600 - - contents_pillar: gnuviechadmin:ssh_known_hosts - - require: - - file: {{ app_home }}/.ssh - - require_in: - - git: {{ gitrepo }} - -SSH configuration: - file.managed: - - name: {{ app_home }}/.ssh/config - - user: {{ app_user }} - - group: {{ app_group }} - - mode: 0600 - - source: salt://gnuviechadmin/ssh_deploy_config - - template: jinja - - context: - key: {{ deployment_key }} - - require: - - file: {{ app_home }}/.ssh - - require_in: - - git: {{ gitrepo }} -{% endif %} + - group: {{ gvaappname }}-group {{ checkout }}: file.directory: @@ -106,6 +68,7 @@ SSH configuration: - user: {{ app_user }} - target: {{ checkout }} - rev: {{ salt['pillar.get']('gnuviechadmin:{}:git_branch'.format(gvaappname), 'production') }} + - force_clone: true - require: - file: {{ checkout }} - watch_in: @@ -113,55 +76,17 @@ SSH configuration: - service: {{ servicename }} {% endif %} -rm -rf {{ venv }}: - cmd.run: - - runas: {{ app_user }} - - unless: test -f {{ venv }}/bin/python3 - - require: - - user: {{ gvaappname }}-user - -create-{{ gvaappname }}-venv: - cmd.run: - - name: python3 -m virtualenv --python=python3 {{ venv }} - - runas: {{ app_user }} - - unless: test -f {{ venv }}/bin/pip3 - - require: - - user: {{ gvaappname }}-user - - python3-virtualenv-packages - - watch_in: - - cmd: update-{{ gvaappname }}-pip - -update-{{ gvaappname }}-pip: - cmd.wait: - - name: {{ venv }}/bin/python3 -m pip install -U pip - - runas: {{ app_user }} - - require: - - user: {{ gvaappname }}-user - -{{ venv }}: - file.directory: - - user: {{ app_user }} - - group: {{ app_group }} - - require: - - cmd: create-{{ gvaappname }}-venv - - watch_in: - - cmd: {{ gvaappname }}-requirements - {{ gvaappname }}-requirements: cmd.wait: - - name: /usr/local/bin/pipenv install --deploy + - name: /usr/local/poetry/bin/poetry install - runas: {{ app_user }} - cwd: {{ checkout }} - env: - - VIRTUAL_ENV: "{{ venv }}" - - PIPENV_HIDE_EMOJIS: 1 - - PIPENV_NOSPIN: 1 - - PIPENV_COLORBLIND: 1 + - POETRY_VIRTUALENVS_IN_PROJECT: "true" - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - require: - - cmd: install_pipenv - - file: {{ venv }} + - cmd: install_poetry {%- if update_git %} - git: {{ gitrepo }} {%- else %} diff --git a/states/python/poetry.sls b/states/python/poetry.sls new file mode 100644 index 0000000..6bb3121 --- /dev/null +++ b/states/python/poetry.sls @@ -0,0 +1,22 @@ +--- +curl: + pkg.installed + +python3: + pkg.installed + +poetry-preconditions: + pkg.installed: + - pkgs: + - python3-wheel + - python3-pip + - python3-setuptools + +install_poetry: + cmd.run: + - name: curl -sSL https://install.python-poetry.org | POETRY_VERSION={{ salt['grains.get']('python:poetry:version', '1.4.2') }} POETRY_HOME=/usr/local/poetry /usr/bin/python3 - + - creates: /usr/local/poetry/bin/poetry + - require: + - id: poetry-preconditions + - pkg: curl + - pkg: python3 diff --git a/states/uwsgi/init.sls b/states/uwsgi/init.sls index ee85dbc..d90e450 100644 --- a/states/uwsgi/init.sls +++ b/states/uwsgi/init.sls @@ -1,8 +1,9 @@ +--- uwsgi: pkg: - installed service.running: - - enable: True - - reload: True + - enable: true + - reload: false - require: - pkg: uwsgi