Make gnuviechadmin components work with Poetry

2023-05-07 13:15:45 +02:00 · 2023-05-07 13:15:45 +02:00 · 1932c76423
parent 18f0061ee4
commit 1932c76423
18 changed files with 41 additions and 29 deletions
--- a/pillar/gnuviechadmin/gvafile.sls
+++ b/pillar/gnuviechadmin/gvafile.sls
@ -9,7 +9,7 @@ gnuviechadmin:
    amqp_user: file
    celery_module: fileservertasks
    fullname: File Server
-    git_branch: master
+    git_branch: main
    git_url: https://git.dittberner.info/gnuviech/gvafile.git
    mail_directory: /home/mail
    sftp_authkeys_directory: /srv/sftp/authorized_keys
--- a/pillar/gnuviechadmin/gvaldap.sls
+++ b/pillar/gnuviechadmin/gvaldap.sls
@ -13,7 +13,7 @@ gnuviechadmin:
    celery_module: ldaptasks
    django_secret_key: IyOiTDt2DMo4gBVTwZ+E2p+mI1S/rNzZVIFlSr6TpgtxtsJODOVWHaxgVW3FqGZVaFU=
    fullname: LDAP
-    git_branch: master
+    git_branch: main
    git_url: https://git.dittberner.info/gnuviech/gvaldap.git
    ldap_admin_password: NnVnGoWBVw6BKb9DhTwHAz0ICrdiDy+HL1A6F2Rz
    ldap_admin_user: ldapadmin
--- a/pillar/gnuviechadmin/gvamysql.sls
+++ b/pillar/gnuviechadmin/gvamysql.sls
@ -9,7 +9,9 @@ gnuviechadmin:
    amqp_user: mysql
    celery_module: mysqltasks
    fullname: MySQL Server
-    git_branch: master
+    git_branch: main
    git_url: https://git.dittberner.info/gnuviech/gvamysql.git
    mysql_admin_password: jSXstgT/AbWofdI2tJWYpQvFX1mtxt4tFMlrYxSA
    mysql_admin_user: gvamysql
+
+mysql.default_file: /etc/mysql/debian.cnf
--- a/pillar/gnuviechadmin/gvapgsql.sls
+++ b/pillar/gnuviechadmin/gvapgsql.sls
@ -9,7 +9,7 @@ gnuviechadmin:
    amqp_user: pgsql
    celery_module: pgsqltasks
    fullname: PostgreSQL Server
-    git_branch: master
+    git_branch: main
    git_url: https://git.dittberner.info/gnuviech/gvapgsql.git
    pgsql_admin_password: AAv6d1t9p/vtX/kVorim2MJROQfQPWJoZP3mzyMW
    pgsql_admin_user: gvapgsql
--- a/pillar/gnuviechadmin/gvaweb.sls
+++ b/pillar/gnuviechadmin/gvaweb.sls
@ -9,5 +9,5 @@ gnuviechadmin:
    amqp_user: web
    celery_module: webtasks
    fullname: Web
-    git_branch: master
+    git_branch: main
    git_url: https://git.dittberner.info/gnuviech/gvaweb.git
--- a/states/gnuviechadmin/celery-worker.service
+++ b/states/gnuviechadmin/celery-worker.service
@ -3,7 +3,7 @@ Description={{ description }}

 [Service]
 EnvironmentFile=-/etc/default/{{ appname }}
-ExecStart={{ virtualenv }}/bin/celery worker -A {{ celery_module }} -Q {{ amqpname }} --loglevel=INFO
+ExecStart={{ checkout }}/.venv/bin/celery --app="{{ celery_module }}" worker --queues="{{ amqpname }}" --task-events --concurrency=1 --loglevel=INFO
 Restart=on-failure
 User={{ app_user }}
 WorkingDirectory={{ checkout }}/{{ appname }}
--- a/states/gnuviechadmin/gvaapp_macros.sls
+++ b/states/gnuviechadmin/gvaapp_macros.sls
@ -1,3 +1,4 @@
+---
 {% macro gvaapp_base(gvaappname, servicename) -%}
 {% set app_home = salt['grains.get']('gnuviechadmin:home', '/home/{}'.format(gvaappname)) %}
 {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %}
@ -7,7 +8,6 @@
 {% set update_git = salt['grains.get']('gnuviechadmin:update_git', True) %}
 {% set gitrepo = salt['pillar.get']('gnuviechadmin:{}:git_url'.format(gvaappname), 'git:gnuviech/{}.git'.format(gvaappname)) -%}
 {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%}
-{% set venv = "{}/.venv".format(checkout) -%}
 {% set deployment_key = '{}/.ssh/id_deployment'.format(app_home) -%}

 {% for host in salt['pillar.get']('gnuviechadmin:machines', {}) %}
@ -77,7 +77,7 @@ gvabase-dependencies:
 {% endif %}

 {{ gvaappname }}-requirements:
-  cmd.wait:
+  cmd.run:
    - name: /usr/local/poetry/bin/poetry install
    - runas: {{ app_user }}
    - cwd: {{ checkout }}
@ -93,7 +93,7 @@ gvabase-dependencies:
      - file: {{ checkout }}
      {%- endif %}
      - pkg: gvabase-dependencies
-    - unless: test $(find {{ venv }} -type f -cnewer Pipfile.lock \! -name '*.pyc'|wc -l) -gt 0
+    - unless: test $(find {{ checkout }}/.venv -type f -cnewer poetry.lock \! -name '*.pyc'|wc -l) -gt 0
    - watch_in:
      - service: {{ servicename }}
 {% endmacro %}
@ -103,7 +103,6 @@ gvabase-dependencies:
 {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %}
 {% set app_group = salt['grains.get']('gnuviechadmin:group', gvaappname) %}

-{% set venv = "{}/{}-venv".format(app_home, gvaappname) -%}
 {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%}
 {% set gitrepo = salt['pillar.get']('gnuviechadmin:{}:git_url'.format(gvaappname), 'git:gnuviech/{}.git'.format(gvaappname)) -%}
 {% set update_git = salt['grains.get']('gnuviechadmin:update_git', True) %}
@ -119,7 +118,6 @@ gvabase-dependencies:
    - source: salt://gnuviechadmin/{{ gvaappname }}/celery-worker.env
    - template: jinja
    - context:
-        virtualenv: {{ venv }}
        checkout: {{ checkout }}
        broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }}
        result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0
@ -134,7 +132,6 @@ gvabase-dependencies:
    - source: salt://gnuviechadmin/celery-worker.service
    - template: jinja
    - context:
-        virtualenv: {{ venv }}
        checkout: {{ checkout }}
        app_user: {{ app_user }}
        appname: {{ gvaappname }}
@ -148,7 +145,6 @@ gvabase-dependencies:
  service.running:
    - enable: True
    - require:
-      - file: {{ venv }}
      {%- if update_git %}
      - git: {{ gitrepo }}
      {%- else %}
--- a/states/gnuviechadmin/gvafile.sls
+++ b/states/gnuviechadmin/gvafile.sls
@ -7,8 +7,7 @@
 {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %}
 include:
  - base
-  - python.pipenv
-  - python.virtualenv
+  - python.poetry
  - nfsserver

 {{ mail_directory }}:
--- a/states/gnuviechadmin/gvafile/celery-worker.env
+++ b/states/gnuviechadmin/gvafile/celery-worker.env
@ -3,3 +3,6 @@ GVAFILE_MAIL_DIRECTORY="{{ salt['pillar.get']('gnuviechadmin:gvafile:mail_direct
 GVAFILE_RESULTS_REDIS_URL="{{ result_url }}"
 GVAFILE_SFTP_AUTHKEYS_DIRECTORY="{{ salt['pillar.get']('gnuviechadmin:gvafile:sftp_authkeys_directory') }}"
 GVAFILE_SFTP_DIRECTORY="{{ salt['pillar.get']('gnuviechadmin:gvafile:web_directory') }}"
+LANG=C.UTF-8
+LC_ALL=C.UTF-8
+TZ=Europe/Berlin
--- a/states/gnuviechadmin/gvaldap.sls
+++ b/states/gnuviechadmin/gvaldap.sls
@ -1,15 +1,16 @@
+---
 {% set gvaappname = salt['pillar.get']('gnuviechadmin:appname') %}
 {% set purpose = "for LDAP data management" %}
 {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %}
 include:
- python.pipenv
- python.virtualenv
+- python.poetry

 {{ create_celery_worker(gvaappname, purpose) }}

 {{ gvaappname }}-dependencies:
  pkg.installed:
    - pkgs:
+      - python3-dev
      - libldap2-dev
      - libsasl2-dev
    - require_in:
--- a/states/gnuviechadmin/gvaldap/celery-worker.env
+++ b/states/gnuviechadmin/gvaldap/celery-worker.env
@ -5,9 +5,12 @@ GVALDAP_ALLOWED_HOSTS="{{ salt['pillar.get']('gnuviechadmin:gvaldap:allowed_host
 GVALDAP_BASEDN_GROUP="{{ salt['pillar.get']('gnuviechadmin:ldap_base_dn_groups') }}"
 GVALDAP_BASEDN_USER="{{ salt['pillar.get']('gnuviechadmin:ldap_base_dn_users') }}"
 GVALDAP_BROKER_URL="{{ broker_url }}"
-GVALDAP_RESULTS_REDIS_URL="{{ result_url }}"
 GVALDAP_LDAP_PASSWORD="{{ salt['pillar.get']('gnuviechadmin:gvaldap:ldap_admin_password' ) }}"
 GVALDAP_LDAP_URL="{{ salt['pillar.get']('gnuviechadmin:ldap_url') }}"
 GVALDAP_LDAP_USER="cn={{ salt['pillar.get']('gnuviechadmin:gvaldap:ldap_admin_user') }},{{ salt['pillar.get']('gnuviechadmin:ldap_base_dn') }}"
+GVALDAP_RESULTS_REDIS_URL="{{ result_url }}"
 GVALDAP_SECRETKEY="{{ salt['pillar.get']('gnuviechadmin:gvaldap:django_secret_key') }}"
 GVALDAP_SERVER_EMAIL="{{ salt['pillar.get']('gnuviechadmin:server_email') }}"
+LANG=C.UTF-8
+LC_ALL=C.UTF-8
+TZ=Europe/Berlin
--- a/states/gnuviechadmin/gvamysql.sls
+++ b/states/gnuviechadmin/gvamysql.sls
@ -6,9 +6,7 @@
 # FIXME: this is broken due to https://github.com/saltstack/salt/issues/56124
 # patches.mysql is needed to be run on the minion before usage with Salt 3000
 include:
- python.pipenv
- python.virtualenv
- patches.mysql
+- python.poetry
 - mariadb-server

 {{ create_celery_worker(gvaappname, purpose) }}
@ -16,21 +14,22 @@ include:
 {{ gvaappname }}-dependencies:
  pkg.installed:
    - pkgs:
+      - python3-dev
      - libmariadb-dev-compat
    - require_in:
      - cmd: {{ gvaappname }}-requirements

 python3-mysqldb:
-  pkg.installed
+  pip.installed:
+    - name: mysqlclient

 gvamysql-mysql-user:
  mysql_user.present:
    - name: {{ mysql_admin_user }}
    - host: '%'
    - password: {{ mysql_admin_password }}
-    - unix_socket: true
    - require:
-      - pkg: python3-mysqldb
+      - pip: python3-mysqldb

 gvamysql-grants-all-dbs:
  mysql_grants.present:
@ -41,4 +40,5 @@ gvamysql-grants-all-dbs:
    - password: {{ mysql_admin_password }}
    - host: '%'
    - require:
+      - pip: python3-mysqldb
      - mysql_user: gvamysql-mysql-user
--- a/states/gnuviechadmin/gvamysql/celery-worker.env
+++ b/states/gnuviechadmin/gvamysql/celery-worker.env
@ -4,3 +4,6 @@ GVAMYSQL_DBADMIN_PASSWORD="{{ salt['pillar.get']('gnuviechadmin:gvamysql:mysql_a
 GVAMYSQL_DBADMIN_PORT="{{ salt['pillar.get']('gnuviechadmin:gvamysql:mysql_admin_port', 3306) }}"
 GVAMYSQL_DBADMIN_USER="{{ salt['pillar.get']('gnuviechadmin:gvamysql:mysql_admin_user', 'gvamysql') }}"
 GVAMYSQL_RESULTS_REDIS_URL="{{ result_url }}"
+LANG=C.UTF-8
+LC_ALL=C.UTF-8
+TZ=Europe/Berlin
--- a/states/gnuviechadmin/gvapgsql.sls
+++ b/states/gnuviechadmin/gvapgsql.sls
@ -2,8 +2,7 @@
 {% set purpose = "for PostgreSQL configuration management" %}
 {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %}
 include:
- python.pipenv
- python.virtualenv
+- python.poetry
 - postgresql-server

 {{ create_celery_worker(gvaappname, purpose) }}
@ -11,6 +10,7 @@ include:
 {{ gvaappname }}-dependencies:
  pkg.installed:
    - pkgs:
+      - python3-dev
      - libpq-dev
    - require_in:
      - cmd: {{ gvaappname }}-requirements
--- a/states/gnuviechadmin/gvapgsql/celery-worker.env
+++ b/states/gnuviechadmin/gvapgsql/celery-worker.env
@ -4,3 +4,6 @@ GVAPGSQL_DBADMIN_HOST="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin
 GVAPGSQL_DBADMIN_PORT="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin_port', 5432) }}"
 GVAPGSQL_DBADMIN_USER="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin_user', 'gvapgsql') }}"
 GVAPGSQL_DBADMIN_PASSWORD="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin_password') }}"
+LANG=C.UTF-8
+LC_ALL=C.UTF-8
+TZ=Europe/Berlin
--- a/states/gnuviechadmin/gvaweb.sls
+++ b/states/gnuviechadmin/gvaweb.sls
@ -3,8 +3,7 @@
 {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %}
 include:
 - base
- python.pipenv
- python.virtualenv
+- python.poetry

 {{ create_celery_worker(gvaappname, purpose) }}

--- a/states/gnuviechadmin/gvaweb/celery-worker.env
+++ b/states/gnuviechadmin/gvaweb/celery-worker.env
@ -3,3 +3,6 @@ GVAWEB_NGINX_SITES_AVAILABLE="{{ salt['pillar.get']('gnuviechadmin:gvaweb:nginx_
 GVAWEB_NGINX_SITES_ENABLED="{{ salt['pillar.get']('gnuviechadmin:gvaweb:nginx_sites_enabled', '/etc/nginx/sites-enabled') }}"
 GVAWEB_RESULTS_REDIS_URL="{{ result_url }}"
 GVAWEB_WWWUSER_MOUNT="{{ salt['pillar.get']('gnuviechadmin:gvaweb:wwwuser_mount', '/srv/wwwfiles') }}"
+LANG=C.UTF-8
+LC_ALL=C.UTF-8
+TZ=Europe/Berlin
--- a/states/mariadb-server/init.sls
+++ b/states/mariadb-server/init.sls
@ -9,7 +9,7 @@ mysql:

 /etc/mysql/my.cnf:
  file.managed:
-    - owner: root
+    - user: root
    - group: root
    - mode: 0644
    - source: salt://mariadb-server/my.cnf