From 1932c76423ab469e86519effaa893b66140ddcd4 Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Sun, 7 May 2023 13:15:45 +0200 Subject: [PATCH] Make gnuviechadmin components work with Poetry --- pillar/gnuviechadmin/gvafile.sls | 2 +- pillar/gnuviechadmin/gvaldap.sls | 2 +- pillar/gnuviechadmin/gvamysql.sls | 4 +++- pillar/gnuviechadmin/gvapgsql.sls | 2 +- pillar/gnuviechadmin/gvaweb.sls | 2 +- states/gnuviechadmin/celery-worker.service | 2 +- states/gnuviechadmin/gvaapp_macros.sls | 10 +++------- states/gnuviechadmin/gvafile.sls | 3 +-- states/gnuviechadmin/gvafile/celery-worker.env | 3 +++ states/gnuviechadmin/gvaldap.sls | 5 +++-- states/gnuviechadmin/gvaldap/celery-worker.env | 5 ++++- states/gnuviechadmin/gvamysql.sls | 12 ++++++------ states/gnuviechadmin/gvamysql/celery-worker.env | 3 +++ states/gnuviechadmin/gvapgsql.sls | 4 ++-- states/gnuviechadmin/gvapgsql/celery-worker.env | 3 +++ states/gnuviechadmin/gvaweb.sls | 3 +-- states/gnuviechadmin/gvaweb/celery-worker.env | 3 +++ states/mariadb-server/init.sls | 2 +- 18 files changed, 41 insertions(+), 29 deletions(-) diff --git a/pillar/gnuviechadmin/gvafile.sls b/pillar/gnuviechadmin/gvafile.sls index 7f74c1f..d47458a 100644 --- a/pillar/gnuviechadmin/gvafile.sls +++ b/pillar/gnuviechadmin/gvafile.sls @@ -9,7 +9,7 @@ gnuviechadmin: amqp_user: file celery_module: fileservertasks fullname: File Server - git_branch: master + git_branch: main git_url: https://git.dittberner.info/gnuviech/gvafile.git mail_directory: /home/mail sftp_authkeys_directory: /srv/sftp/authorized_keys diff --git a/pillar/gnuviechadmin/gvaldap.sls b/pillar/gnuviechadmin/gvaldap.sls index 2df1349..8dedbce 100644 --- a/pillar/gnuviechadmin/gvaldap.sls +++ b/pillar/gnuviechadmin/gvaldap.sls @@ -13,7 +13,7 @@ gnuviechadmin: celery_module: ldaptasks django_secret_key: IyOiTDt2DMo4gBVTwZ+E2p+mI1S/rNzZVIFlSr6TpgtxtsJODOVWHaxgVW3FqGZVaFU= fullname: LDAP - git_branch: master + git_branch: main git_url: https://git.dittberner.info/gnuviech/gvaldap.git ldap_admin_password: NnVnGoWBVw6BKb9DhTwHAz0ICrdiDy+HL1A6F2Rz ldap_admin_user: ldapadmin diff --git a/pillar/gnuviechadmin/gvamysql.sls b/pillar/gnuviechadmin/gvamysql.sls index 938a6d1..97f58a1 100644 --- a/pillar/gnuviechadmin/gvamysql.sls +++ b/pillar/gnuviechadmin/gvamysql.sls @@ -9,7 +9,9 @@ gnuviechadmin: amqp_user: mysql celery_module: mysqltasks fullname: MySQL Server - git_branch: master + git_branch: main git_url: https://git.dittberner.info/gnuviech/gvamysql.git mysql_admin_password: jSXstgT/AbWofdI2tJWYpQvFX1mtxt4tFMlrYxSA mysql_admin_user: gvamysql + +mysql.default_file: /etc/mysql/debian.cnf diff --git a/pillar/gnuviechadmin/gvapgsql.sls b/pillar/gnuviechadmin/gvapgsql.sls index 1fa58d7..3c4ba36 100644 --- a/pillar/gnuviechadmin/gvapgsql.sls +++ b/pillar/gnuviechadmin/gvapgsql.sls @@ -9,7 +9,7 @@ gnuviechadmin: amqp_user: pgsql celery_module: pgsqltasks fullname: PostgreSQL Server - git_branch: master + git_branch: main git_url: https://git.dittberner.info/gnuviech/gvapgsql.git pgsql_admin_password: AAv6d1t9p/vtX/kVorim2MJROQfQPWJoZP3mzyMW pgsql_admin_user: gvapgsql diff --git a/pillar/gnuviechadmin/gvaweb.sls b/pillar/gnuviechadmin/gvaweb.sls index 8aa9c86..81ae841 100644 --- a/pillar/gnuviechadmin/gvaweb.sls +++ b/pillar/gnuviechadmin/gvaweb.sls @@ -9,5 +9,5 @@ gnuviechadmin: amqp_user: web celery_module: webtasks fullname: Web - git_branch: master + git_branch: main git_url: https://git.dittberner.info/gnuviech/gvaweb.git diff --git a/states/gnuviechadmin/celery-worker.service b/states/gnuviechadmin/celery-worker.service index 6963b92..c9506c8 100644 --- a/states/gnuviechadmin/celery-worker.service +++ b/states/gnuviechadmin/celery-worker.service @@ -3,7 +3,7 @@ Description={{ description }} [Service] EnvironmentFile=-/etc/default/{{ appname }} -ExecStart={{ virtualenv }}/bin/celery worker -A {{ celery_module }} -Q {{ amqpname }} --loglevel=INFO +ExecStart={{ checkout }}/.venv/bin/celery --app="{{ celery_module }}" worker --queues="{{ amqpname }}" --task-events --concurrency=1 --loglevel=INFO Restart=on-failure User={{ app_user }} WorkingDirectory={{ checkout }}/{{ appname }} diff --git a/states/gnuviechadmin/gvaapp_macros.sls b/states/gnuviechadmin/gvaapp_macros.sls index 85b196f..3490c1f 100644 --- a/states/gnuviechadmin/gvaapp_macros.sls +++ b/states/gnuviechadmin/gvaapp_macros.sls @@ -1,3 +1,4 @@ +--- {% macro gvaapp_base(gvaappname, servicename) -%} {% set app_home = salt['grains.get']('gnuviechadmin:home', '/home/{}'.format(gvaappname)) %} {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %} @@ -7,7 +8,6 @@ {% set update_git = salt['grains.get']('gnuviechadmin:update_git', True) %} {% set gitrepo = salt['pillar.get']('gnuviechadmin:{}:git_url'.format(gvaappname), 'git:gnuviech/{}.git'.format(gvaappname)) -%} {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%} -{% set venv = "{}/.venv".format(checkout) -%} {% set deployment_key = '{}/.ssh/id_deployment'.format(app_home) -%} {% for host in salt['pillar.get']('gnuviechadmin:machines', {}) %} @@ -77,7 +77,7 @@ gvabase-dependencies: {% endif %} {{ gvaappname }}-requirements: - cmd.wait: + cmd.run: - name: /usr/local/poetry/bin/poetry install - runas: {{ app_user }} - cwd: {{ checkout }} @@ -93,7 +93,7 @@ gvabase-dependencies: - file: {{ checkout }} {%- endif %} - pkg: gvabase-dependencies - - unless: test $(find {{ venv }} -type f -cnewer Pipfile.lock \! -name '*.pyc'|wc -l) -gt 0 + - unless: test $(find {{ checkout }}/.venv -type f -cnewer poetry.lock \! -name '*.pyc'|wc -l) -gt 0 - watch_in: - service: {{ servicename }} {% endmacro %} @@ -103,7 +103,6 @@ gvabase-dependencies: {% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %} {% set app_group = salt['grains.get']('gnuviechadmin:group', gvaappname) %} -{% set venv = "{}/{}-venv".format(app_home, gvaappname) -%} {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%} {% set gitrepo = salt['pillar.get']('gnuviechadmin:{}:git_url'.format(gvaappname), 'git:gnuviech/{}.git'.format(gvaappname)) -%} {% set update_git = salt['grains.get']('gnuviechadmin:update_git', True) %} @@ -119,7 +118,6 @@ gvabase-dependencies: - source: salt://gnuviechadmin/{{ gvaappname }}/celery-worker.env - template: jinja - context: - virtualenv: {{ venv }} checkout: {{ checkout }} broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }} result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0 @@ -134,7 +132,6 @@ gvabase-dependencies: - source: salt://gnuviechadmin/celery-worker.service - template: jinja - context: - virtualenv: {{ venv }} checkout: {{ checkout }} app_user: {{ app_user }} appname: {{ gvaappname }} @@ -148,7 +145,6 @@ gvabase-dependencies: service.running: - enable: True - require: - - file: {{ venv }} {%- if update_git %} - git: {{ gitrepo }} {%- else %} diff --git a/states/gnuviechadmin/gvafile.sls b/states/gnuviechadmin/gvafile.sls index baa8dcb..6fd914a 100644 --- a/states/gnuviechadmin/gvafile.sls +++ b/states/gnuviechadmin/gvafile.sls @@ -7,8 +7,7 @@ {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %} include: - base - - python.pipenv - - python.virtualenv + - python.poetry - nfsserver {{ mail_directory }}: diff --git a/states/gnuviechadmin/gvafile/celery-worker.env b/states/gnuviechadmin/gvafile/celery-worker.env index 31b3665..acbf36d 100644 --- a/states/gnuviechadmin/gvafile/celery-worker.env +++ b/states/gnuviechadmin/gvafile/celery-worker.env @@ -3,3 +3,6 @@ GVAFILE_MAIL_DIRECTORY="{{ salt['pillar.get']('gnuviechadmin:gvafile:mail_direct GVAFILE_RESULTS_REDIS_URL="{{ result_url }}" GVAFILE_SFTP_AUTHKEYS_DIRECTORY="{{ salt['pillar.get']('gnuviechadmin:gvafile:sftp_authkeys_directory') }}" GVAFILE_SFTP_DIRECTORY="{{ salt['pillar.get']('gnuviechadmin:gvafile:web_directory') }}" +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +TZ=Europe/Berlin diff --git a/states/gnuviechadmin/gvaldap.sls b/states/gnuviechadmin/gvaldap.sls index b7da1f0..12a1b54 100644 --- a/states/gnuviechadmin/gvaldap.sls +++ b/states/gnuviechadmin/gvaldap.sls @@ -1,15 +1,16 @@ +--- {% set gvaappname = salt['pillar.get']('gnuviechadmin:appname') %} {% set purpose = "for LDAP data management" %} {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %} include: -- python.pipenv -- python.virtualenv +- python.poetry {{ create_celery_worker(gvaappname, purpose) }} {{ gvaappname }}-dependencies: pkg.installed: - pkgs: + - python3-dev - libldap2-dev - libsasl2-dev - require_in: diff --git a/states/gnuviechadmin/gvaldap/celery-worker.env b/states/gnuviechadmin/gvaldap/celery-worker.env index ce57156..c0d580b 100644 --- a/states/gnuviechadmin/gvaldap/celery-worker.env +++ b/states/gnuviechadmin/gvaldap/celery-worker.env @@ -5,9 +5,12 @@ GVALDAP_ALLOWED_HOSTS="{{ salt['pillar.get']('gnuviechadmin:gvaldap:allowed_host GVALDAP_BASEDN_GROUP="{{ salt['pillar.get']('gnuviechadmin:ldap_base_dn_groups') }}" GVALDAP_BASEDN_USER="{{ salt['pillar.get']('gnuviechadmin:ldap_base_dn_users') }}" GVALDAP_BROKER_URL="{{ broker_url }}" -GVALDAP_RESULTS_REDIS_URL="{{ result_url }}" GVALDAP_LDAP_PASSWORD="{{ salt['pillar.get']('gnuviechadmin:gvaldap:ldap_admin_password' ) }}" GVALDAP_LDAP_URL="{{ salt['pillar.get']('gnuviechadmin:ldap_url') }}" GVALDAP_LDAP_USER="cn={{ salt['pillar.get']('gnuviechadmin:gvaldap:ldap_admin_user') }},{{ salt['pillar.get']('gnuviechadmin:ldap_base_dn') }}" +GVALDAP_RESULTS_REDIS_URL="{{ result_url }}" GVALDAP_SECRETKEY="{{ salt['pillar.get']('gnuviechadmin:gvaldap:django_secret_key') }}" GVALDAP_SERVER_EMAIL="{{ salt['pillar.get']('gnuviechadmin:server_email') }}" +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +TZ=Europe/Berlin diff --git a/states/gnuviechadmin/gvamysql.sls b/states/gnuviechadmin/gvamysql.sls index 6ae2ecb..5b93c8a 100644 --- a/states/gnuviechadmin/gvamysql.sls +++ b/states/gnuviechadmin/gvamysql.sls @@ -6,9 +6,7 @@ # FIXME: this is broken due to https://github.com/saltstack/salt/issues/56124 # patches.mysql is needed to be run on the minion before usage with Salt 3000 include: -- python.pipenv -- python.virtualenv -- patches.mysql +- python.poetry - mariadb-server {{ create_celery_worker(gvaappname, purpose) }} @@ -16,21 +14,22 @@ include: {{ gvaappname }}-dependencies: pkg.installed: - pkgs: + - python3-dev - libmariadb-dev-compat - require_in: - cmd: {{ gvaappname }}-requirements python3-mysqldb: - pkg.installed + pip.installed: + - name: mysqlclient gvamysql-mysql-user: mysql_user.present: - name: {{ mysql_admin_user }} - host: '%' - password: {{ mysql_admin_password }} - - unix_socket: true - require: - - pkg: python3-mysqldb + - pip: python3-mysqldb gvamysql-grants-all-dbs: mysql_grants.present: @@ -41,4 +40,5 @@ gvamysql-grants-all-dbs: - password: {{ mysql_admin_password }} - host: '%' - require: + - pip: python3-mysqldb - mysql_user: gvamysql-mysql-user diff --git a/states/gnuviechadmin/gvamysql/celery-worker.env b/states/gnuviechadmin/gvamysql/celery-worker.env index 1fdf273..ec3a2e3 100644 --- a/states/gnuviechadmin/gvamysql/celery-worker.env +++ b/states/gnuviechadmin/gvamysql/celery-worker.env @@ -4,3 +4,6 @@ GVAMYSQL_DBADMIN_PASSWORD="{{ salt['pillar.get']('gnuviechadmin:gvamysql:mysql_a GVAMYSQL_DBADMIN_PORT="{{ salt['pillar.get']('gnuviechadmin:gvamysql:mysql_admin_port', 3306) }}" GVAMYSQL_DBADMIN_USER="{{ salt['pillar.get']('gnuviechadmin:gvamysql:mysql_admin_user', 'gvamysql') }}" GVAMYSQL_RESULTS_REDIS_URL="{{ result_url }}" +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +TZ=Europe/Berlin diff --git a/states/gnuviechadmin/gvapgsql.sls b/states/gnuviechadmin/gvapgsql.sls index 10d752d..26443a4 100644 --- a/states/gnuviechadmin/gvapgsql.sls +++ b/states/gnuviechadmin/gvapgsql.sls @@ -2,8 +2,7 @@ {% set purpose = "for PostgreSQL configuration management" %} {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %} include: -- python.pipenv -- python.virtualenv +- python.poetry - postgresql-server {{ create_celery_worker(gvaappname, purpose) }} @@ -11,6 +10,7 @@ include: {{ gvaappname }}-dependencies: pkg.installed: - pkgs: + - python3-dev - libpq-dev - require_in: - cmd: {{ gvaappname }}-requirements diff --git a/states/gnuviechadmin/gvapgsql/celery-worker.env b/states/gnuviechadmin/gvapgsql/celery-worker.env index 21bc468..63289e4 100644 --- a/states/gnuviechadmin/gvapgsql/celery-worker.env +++ b/states/gnuviechadmin/gvapgsql/celery-worker.env @@ -4,3 +4,6 @@ GVAPGSQL_DBADMIN_HOST="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin GVAPGSQL_DBADMIN_PORT="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin_port', 5432) }}" GVAPGSQL_DBADMIN_USER="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin_user', 'gvapgsql') }}" GVAPGSQL_DBADMIN_PASSWORD="{{ salt['pillar.get']('gnuviechadmin:gvapgsql:pgsql_admin_password') }}" +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +TZ=Europe/Berlin diff --git a/states/gnuviechadmin/gvaweb.sls b/states/gnuviechadmin/gvaweb.sls index 873cf29..a301e89 100644 --- a/states/gnuviechadmin/gvaweb.sls +++ b/states/gnuviechadmin/gvaweb.sls @@ -3,8 +3,7 @@ {% from 'gnuviechadmin/gvaapp_macros.sls' import create_celery_worker with context %} include: - base -- python.pipenv -- python.virtualenv +- python.poetry {{ create_celery_worker(gvaappname, purpose) }} diff --git a/states/gnuviechadmin/gvaweb/celery-worker.env b/states/gnuviechadmin/gvaweb/celery-worker.env index 31b01f1..67763b2 100644 --- a/states/gnuviechadmin/gvaweb/celery-worker.env +++ b/states/gnuviechadmin/gvaweb/celery-worker.env @@ -3,3 +3,6 @@ GVAWEB_NGINX_SITES_AVAILABLE="{{ salt['pillar.get']('gnuviechadmin:gvaweb:nginx_ GVAWEB_NGINX_SITES_ENABLED="{{ salt['pillar.get']('gnuviechadmin:gvaweb:nginx_sites_enabled', '/etc/nginx/sites-enabled') }}" GVAWEB_RESULTS_REDIS_URL="{{ result_url }}" GVAWEB_WWWUSER_MOUNT="{{ salt['pillar.get']('gnuviechadmin:gvaweb:wwwuser_mount', '/srv/wwwfiles') }}" +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +TZ=Europe/Berlin diff --git a/states/mariadb-server/init.sls b/states/mariadb-server/init.sls index 51d5ef8..a18f70a 100644 --- a/states/mariadb-server/init.sls +++ b/states/mariadb-server/init.sls @@ -9,7 +9,7 @@ mysql: /etc/mysql/my.cnf: file.managed: - - owner: root + - user: root - group: root - mode: 0644 - source: salt://mariadb-server/my.cnf