Implement salt states for gva webinterface

- setup listener and pg_hba.conf for PostgreSQL server
- add state code for gva
- add macros for nginx and uwsgi with Python 3 support
- add pillar data for gva

pillar/gnuviechadmin/gva.sls

@@ -1,9 +1,18 @@
 include:
   - gnuviechadmin
+  - gnuviechadmin.database
   - gnuviechadmin.queues.common
+  - gnuviechadmin.queues.gva
 
 gnuviechadmin:
   appname: gva
+  database:
+    host: pgsql
   gva:
+    fullname: Self Service Web Interface
     django_secret_key: yBnbG4azhNaTxIW0/Rv2dEij9PcVU1KVR//1bR6LujmLBnZJw8OOrEi2dIqz3pyOdG8=
-
+    git_url: https://git.dittberner.info/gnuviech/gva.git
+    git_branch: master
+    url_webmail: https://webmail.gva.local/
+    url_mysql_admin: https://phpmyadmin.gva.local/
+    url_pgsql_admin: https://phppgadmin.gva.local/

pillar/gnuviechadmin/init.sls

@@ -6,6 +6,7 @@ gnuviechadmin:
       nextgit.gnuviech-server.de ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBESb6Q0nyvx82wJ0S6Jx7ZvY6wJzuwqh2zWOlXzLDcor8Pu5iLqUn5GywS0ooyl3Hkyn983R6Zdr49zgTroRwQA=
   deploymenttype: local
   mailfrom: admin@gnuviech-server.de
+  adminname: Gnuviech Admin
   adminemail: admin@gnuviech-server.de
   sitename: Gnuviech Customer Self Service
   domainname: localhost
@@ -21,18 +22,43 @@ gnuviechadmin:
   ldap_domain: gva.local
   ldap_url: ldap://ldap
   machines:
-    gva.local:
+    salt:
-      ip: 172.16.3.2
+      ip: 172.16.4.10
+    mq:
+      ip: 172.16.4.20
+    syslog:
+      ip: 172.16.4.30
+    pgsql:
+      ip: 172.16.4.40
       names:
-        - mq
+        - pgsql
-        - gva.local
+        - gvapgsql
-    gvaldap.local:
+    dns:
-      ip: 172.16.3.3
+      ip: 172.16.4.50
-    gvafile.local:
+    ldap:
-      ip: 172.16.3.4
+      ip: 172.16.4.60
-    gvaweb.local:
+      names:
-      ip: 172.16.3.5
+        - ldap
-    gvamysql.local:
+        - gvaldap
-      ip: 172.16.3.6
+    file:
-    gvapgsql.local:
+      ip: 172.16.4.70
-      ip: 172.16.3.7
+      names:
+        - file
+        - gvafile
+    mail:
+      ip: 172.16.4.80
+    mysql:
+      ip: 172.16.4.90
+      names:
+        - mysql
+        - gvamysql
+    web:
+      ip: 172.16.4.100
+      names:
+        - web
+        - gvaweb
+    service:
+      ip: 172.16.4.110
+      names:
+        - service
+        - gva

pillar/postgresql-server/init.sls

@@ -0,0 +1,6 @@
+postgresql-server:
+  local-net: 172.16.4.0/24
+  shared_buffers: 128MB
+  work_mem: 5MB
+  maintenance_work_mem: 4MB
+  effective_cache_size: 128MB

pillar/top.sls

@@ -7,7 +7,7 @@ base:
     - match: grain
     - gnuviechadmin.{{ role }}
 {% endfor %}
-{% for role in ('fileserver', 'ldapserver', 'ldapclient') %}
+{% for role in ('fileserver', 'ldapserver', 'ldapclient', 'postgresql-server', 'webserver') %}
   'roles:{{ role }}':
     - match: grain
     - {{ role }}

pillar/webserver/init.sls

@@ -0,0 +1 @@
+

states/gnuviechadmin/gva.sls

@@ -0,0 +1,128 @@
+{% set gvaappname = salt['pillar.get']('gnuviechadmin:appname') %}
+{% set app_home = salt['grains.get']('gnuviechadmin:home', '/home/{}'.format(gvaappname)) %}
+{% set app_user = salt['grains.get']('gnuviechadmin:user', gvaappname) %}
+{% set app_group = salt['grains.get']('gnuviechadmin:group', gvaappname) %}
+{% set venv = "{}/{}-venv".format(app_home, gvaappname) -%}
+
+{% set amqp_user = salt['pillar.get']('gnuviechadmin:{}:amqp_user'.format(gvaappname), gvaappname) -%}
+{% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%}
+{% set domainname = salt['pillar.get']('gnuviechadmin:{}:domainname'.format(gvaappname), 'service.localhost') %}
+{% set update_git = salt['grains.get']('gnuviechadmin:update_git', True) %}
+{% set gitrepo = salt['pillar.get']('gnuviechadmin:{}:git_url'.format(gvaappname), 'git:gnuviech/{}.git'.format(gvaappname)) -%}
+
+{% from 'gnuviechadmin/gvaapp_macros.sls' import gvaapp_base with context %}
+include:
+- base
+- python.pipenv
+- python.virtualenv
+- uwsgi.python3
+
+{{ gvaapp_base(gvaappname, 'uwsgi') }}
+
+{{ gvaappname }}-dependencies:
+  pkg.installed:
+    - pkgs:
+      - libpq-dev
+    - require_in:
+      - cmd: {{ gvaappname }}-requirements
+
+gettext:
+  pkg.installed
+
+{{ checkout }}/.env:
+  file.managed:
+    - user: {{ app_user }}
+    - group: {{ app_group }}
+    - mode: 0640
+    - source: salt://gnuviechadmin/{{ gvaappname }}/env-vars
+    - template: jinja
+    - context:
+        gvaappname: {{ gvaappname }}
+        broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }}
+        result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0
+    - require:
+      - user: {{ gvaappname }}-user
+      - group: {{ gvaappname }}-group
+      - file: {{ checkout }}
+
+{% for command in ['migrate --noinput', 'collectstatic --noinput', 'compilemessages'] %}
+{{ gvaappname }}-manage-{{ command }}:
+  cmd.wait:
+    - name: /usr/local/bin/pipenv run python3 manage.py {{ command }}
+    - runas: {{ app_user }}
+    - cwd: {{ checkout }}/gnuviechadmin
+    - env:
+      - VIRTUAL_ENV: "{{ venv }}"
+      - LC_ALL: C.UTF-8
+      - LANG: C.UTF-8
+    - watch:
+      - cmd: {{ gvaappname }}-requirements
+      - file: {{ checkout }}/.env
+      {%- if update_git %}
+      - git: {{ gitrepo }}
+      {%- endif %}
+{% endfor %}
+
+/etc/uwsgi/apps-available/{{ gvaappname }}.ini:
+  file.managed:
+    - user: root
+    - group: {{ app_group }}
+    - mode: 0640
+    - source: salt://gnuviechadmin/{{ gvaappname }}/uwsgi.ini
+    - template: jinja
+    - context:
+        gvaappname: {{ gvaappname }}
+        broker_url: amqp://{{ amqp_user }}:{{ salt['pillar.get']('gnuviechadmin:queues:users:{}:password'.format(amqp_user)) }}@{{ salt['pillar.get']('gnuviechadmin:amqp_host', 'mq') }}/{{ salt['pillar.get']('gnuviechadmin:queues:vhost') }}
+        result_url: redis://:{{ salt['pillar.get']('gnuviechadmin:redis_password') }}@{{ salt['pillar.get']('gnuviechadmin:redis_host') }}/0
+        workdir: {{ checkout }}/gnuviechadmin
+        venv: {{ venv }}
+    - require:
+      - pkg: uwsgi
+    - require_in:
+      - service: uwsgi
+    - watch_in:
+      - service: uwsgi
+
+/etc/uwsgi/apps-enabled/{{ gvaappname }}.ini:
+  file.symlink:
+    - target: /etc/uwsgi/apps-available/{{ gvaappname }}.ini
+    - require:
+      - file: /etc/uwsgi/apps-available/{{ gvaappname }}.ini
+    - require_in:
+      - service: uwsgi
+
+{% set letsencrypt = salt['pillar.get']('gnuviechadmin:{}:letsencrypt'.format(gvaappname), False) %}
+{% if not letsencrypt %}
+python3-cryptography:
+  pkg.installed
+
+{% from 'webserver/sslcert.macros.sls' import key_cert with context %}
+{{ key_cert(domainname) }}
+{% endif %}
+
+/etc/nginx/sites-available/{{ domainname }}:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 0640
+    - source: salt://gnuviechadmin/{{ gvaappname }}/app.nginx
+    - template: jinja
+    - context:
+        domainname: {{ domainname }}
+        checkout: {{ checkout }}
+        letsencrypt: {{ letsencrypt }}
+        appname: {{ gvaappname }}
+    - require:
+      - pkg: nginx
+    - watch_in:
+      - service: nginx
+
+/etc/nginx/sites-enabled/{{ domainname }}:
+  file.symlink:
+    - target: /etc/nginx/sites-available/{{ domainname }}
+    - require:
+      - file: /etc/nginx/sites-available/{{ domainname }}
+      - file: /etc/uwsgi/apps-enabled/{{ gvaappname }}.ini
+      - service: uwsgi
+    - watch_in:
+      - service: nginx

states/gnuviechadmin/gva/app.nginx

@@ -0,0 +1,32 @@
+{% import "webserver/site_macros.nginx" as nginx with context -%}
+
+{{ nginx.server_definition(domainname, letsencrypt=letsencrypt) }}
+}
+
+{{ nginx.server_definition(domainname, True, letsencrypt=letsencrypt) }}
+  server_name {{ domainname }};
+
+  if ( $host != '{{ domainname }}') {
+    return 301 https://{{ domainname }}$request_uri;
+  }
+
+  client_max_body_size 1M;
+  gzip on;
+  gzip_types text/javascript application/javascript application/x-javascript text/css;
+  add_header Strict-Transport-Security max-age=15552000; # 180 days
+
+  location /media {
+    alias {{ checkout }}/media;
+    expires 10m;
+  }
+
+  location /static {
+    alias {{ checkout }}/static;
+    expires 6M;
+  }
+
+  location / {
+    include uwsgi_params;
+    uwsgi_pass unix:/run/uwsgi/app/{{ appname }}/socket;
+  }
+}

states/gnuviechadmin/gva/env-vars

@@ -0,0 +1,23 @@
+DJANGO_SETTINGS_MODULE=gnuviechadmin.settings
+GVA_ADMIN_EMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }}
+GVA_ADMIN_NAME={{ salt['pillar.get']('gnuviechadmin:adminname', 'Gnuviech Admin') }}
+GVA_BROKER_URL={{ broker_url }}
+GVA_DOMAIN_NAME={{ salt['pillar.get']('gnuviechadmin:{}:domainname'.format(gvaappname), 'service.localhost') }}
+GVA_MIN_OS_GID={{ salt['pillar.get']('gnuviechadmin:minosgid', 10000) }}
+GVA_MIN_OS_UID={{ salt['pillar.get']('gnuviechadmin:minosuid', 10000) }}
+GVA_OSUSER_DEFAULT_SHELL={{ salt['pillar.get']('gnuviechadmin:osuserdefaultshell', '/sbin/nologin') }}
+GVA_OSUSER_HOME_BASEPATH={{ salt['pillar.get']('gnuviechadmin:osuserhomedirbase', '/home') }}
+GVA_OSUSER_PREFIX={{ salt['pillar.get']('gnuviechadmin:osuserprefix', 'user') }}
+GVA_OSUSER_UPLOADSERVER={{ salt['pillar.get']('gnuviechadmin:uploadserver') }}
+GVA_PGSQL_DATABASE={{ salt['pillar.get']('gnuviechadmin:database:name') }}
+GVA_PGSQL_HOSTNAME={{ salt['pillar.get']('gnuviechadmin:database:host', 'localhost') }}
+GVA_PGSQL_PASSWORD={{ salt['pillar.get']('gnuviechadmin:database:owner:password') }}
+GVA_PGSQL_PORT={{ salt['pillar.get']('gnuviechadmin:database:port', 5432) }}
+GVA_PGSQL_USER={{ salt['pillar.get']('gnuviechadmin:database:owner:user', gvaappname ) }}
+GVA_RESULTS_REDIS_URL={{ result_url }}
+GVA_SITE_ADMINMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }}
+GVA_SITE_NAME={{ salt['pillar.get']('gnuviechadmin:sitename') }}
+GVA_SITE_SECRET={{ salt['pillar.get']('gnuviechadmin:{}:django_secret_key'.format(gvaappname)) }}
+GVA_URL_MYSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_mysql_admin'.format(gvaappname)) }}
+GVA_URL_PGSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_pgsql_admin'.format(gvaappname)) }}
+GVA_URL_WEBMAIL={{ salt['pillar.get']('gnuviechadmin:{}:url_webmail'.format(gvaappname)) }}

states/gnuviechadmin/gva/uwsgi.ini

@@ -0,0 +1,35 @@
+[uwsgi]
+chdir = {{ workdir }}
+master = True
+max-requests = 5000
+module = django.core.wsgi:get_wsgi_application()
+plugin = python37
+processes = 4
+threads = 2
+uid = {{ gvaappname }}
+vacuum = True
+virtualenv = {{ venv }}
+
+env = DJANGO_SETTINGS_MODULE=gnuviechadmin.settings
+env = GVA_ADMIN_EMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }}
+env = GVA_ADMIN_NAME={{ salt['pillar.get']('gnuviechadmin:adminname', 'Gnuviech Admin') }}
+env = GVA_BROKER_URL={{ broker_url }}
+env = GVA_DOMAIN_NAME={{ salt['pillar.get']('gnuviechadmin:{}:domainname'.format(gvaappname), 'service.localhost') }}
+env = GVA_MIN_OS_GID={{ salt['pillar.get']('gnuviechadmin:minosgid', 10000) }}
+env = GVA_MIN_OS_UID={{ salt['pillar.get']('gnuviechadmin:minosuid', 10000) }}
+env = GVA_OSUSER_DEFAULT_SHELL={{ salt['pillar.get']('gnuviechadmin:osuserdefaultshell', '/sbin/nologin') }}
+env = GVA_OSUSER_HOME_BASEPATH={{ salt['pillar.get']('gnuviechadmin:osuserhomedirbase', '/home') }}
+env = GVA_OSUSER_PREFIX={{ salt['pillar.get']('gnuviechadmin:osuserprefix', 'user') }}
+env = GVA_OSUSER_UPLOADSERVER={{ salt['pillar.get']('gnuviechadmin:uploadserver') }}
+env = GVA_PGSQL_DATABASE={{ salt['pillar.get']('gnuviechadmin:database:name') }}
+env = GVA_PGSQL_HOSTNAME={{ salt['pillar.get']('gnuviechadmin:database:host', 'localhost') }}
+env = GVA_PGSQL_PASSWORD={{ salt['pillar.get']('gnuviechadmin:database:owner:password') }}
+env = GVA_PGSQL_PORT={{ salt['pillar.get']('gnuviechadmin:database:port', 5432) }}
+env = GVA_PGSQL_USER={{ salt['pillar.get']('gnuviechadmin:database:owner:user', gvaappname ) }}
+env = GVA_RESULTS_REDIS_URL={{ result_url }}
+env = GVA_SITE_ADMINMAIL={{ salt['pillar.get']('gnuviechadmin:adminemail', 'admin@example.org') }}
+env = GVA_SITE_NAME={{ salt['pillar.get']('gnuviechadmin:sitename') }}
+env = GVA_SITE_SECRET={{ salt['pillar.get']('gnuviechadmin:{}:django_secret_key'.format(gvaappname)) }}
+env = GVA_URL_MYSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_mysql_admin'.format(gvaappname)) }}
+env = GVA_URL_PGSQL_ADMIN={{ salt['pillar.get']('gnuviechadmin:{}:url_pgsql_admin'.format(gvaappname)) }}
+env = GVA_URL_WEBMAIL={{ salt['pillar.get']('gnuviechadmin:{}:url_webmail'.format(gvaappname)) }}

states/gnuviechadmin/gvaapp_macros.sls

@@ -10,6 +10,20 @@
 {% set checkout = salt['grains.get']('gnuviechadmin:checkout', '/srv/{}'.format(gvaappname)) -%}
 {% set deployment_key = '{}/.ssh/id_deployment'.format(app_home) -%}
 
+{% for host in salt['pillar.get']('gnuviechadmin:machines', {}) %}
+{% if host != salt['grains.get']('host') %}
+{{ host }}:
+  host.present:
+    - ip: {{ salt['pillar.get']('gnuviechadmin:machines:{}:ip'.format(host)) }}
+{% if salt['pillar.get']('gnuviechadmin:machines:{}:names'.format(host)) %}
+    - names:
+{% for machine in salt['pillar.get']('gnuviechadmin:machines:{}:names'.format(host)) %}
+      - {{ machine }}
+{% endfor %}
+{% endif %}
+{% endif %}
+{% endfor %}
+
 {{ gvaappname }}-group:
   group.present:
     - name: {{ app_group }}
@@ -22,6 +36,8 @@
     - fullname: {{ appfullname }}
     - groups:
       - {{ app_group }}
+    - require:
+      - group: {{ gvaappname }}-group
   alias.present:
     - target: root
 
@@ -169,7 +185,7 @@ update-{{ gvaappname }}-pip:
 
 {% set servicename = "{}-celery-worker".format(gvaappname) %}
 {% set amqp_user = salt['pillar.get']('gnuviechadmin:{}:amqp_user'.format(gvaappname)) -%}
-{{ gvaapp_base(gvaappname, servicename ) }}
+{{ gvaapp_base(gvaappname, servicename) }}
 /etc/default/{{ gvaappname }}:
   file.managed:
     - user: root

states/postgresql-server/custom.conf

@@ -0,0 +1,11 @@
+listen_addresses = '{{ salt['grains.get']('ipv4') | join(",") }}'
+shared_buffers = {{ salt['pillar.get']('postgresql-server:shared_buffers', '1GB') }}
+work_mem = {{ salt['pillar.get']('postgresql-server:work_mem', '10MB') }}
+maintenance_work_mem = {{ salt['pillar.get']('postgresql-server:maintenance_work_mem', '32MB') }}
+effective_cache_size = {{ salt['pillar.get']('postgresql-server:effective_cache_size', '2GB') }}
+
+lc_messages = 'de_DE.UTF-8'			# locale for system error message
+lc_monetary = 'de_DE.UTF-8'			# locale for monetary formatting
+lc_numeric = 'de_DE.UTF-8'			# locale for number formatting
+lc_time = 'de_DE.UTF-8'				# locale for time formatting
+default_text_search_config = 'pg_catalog.german'

states/postgresql-server/init.sls

@@ -7,3 +7,24 @@ postgresql:
   service.running:
     - require:
       - pkg: postgresql
+
+/etc/postgresql/11/main/conf.d/custom.conf:
+  file.managed:
+    - user: postgres
+    - group: postgres
+    - source: salt://postgresql-server/custom.conf
+    - template: jinja
+    - mode: 0644
+    - require:
+      - pkg: postgresql
+    - watch_in:
+      - service: postgresql
+
+/etc/postgresql/11/main/pg_hba.conf:
+  file.append:
+    - source: salt://postgresql-server/pg_hba_line.conf
+    - template: jinja
+    - require:
+      - pkg: postgresql
+    - watch_in:
+      - service: postgresql

states/postgresql-server/pg_hba_line.conf

@@ -0,0 +1 @@
+host    all             all             {{ salt['pillar.get']('postgresql-server:local-net') }}     md5

states/uwsgi/init.sls

@@ -0,0 +1,8 @@
+uwsgi:
+  pkg:
+    - installed
+  service.running:
+    - enable: True
+    - reload: True
+    - require:
+      - pkg: uwsgi

states/uwsgi/python3.sls

@@ -0,0 +1,7 @@
+include:
+  - uwsgi
+
+uwsgi-plugin-python3:
+  pkg.installed:
+    - require_in:
+      - service: uwsgi

states/webserver/site_macros.nginx

@@ -0,0 +1,57 @@
+{#
+macros for nginx configuration files
+#}
+{% macro logfiles(server_name, ssl=False) -%}
+  access_log {{ salt['pillar.get']('nginx:logdir', '/var/log/nginx') }}/{{ server_name }}{% if ssl %}-ssl{% endif %}.access.log;
+  error_log {{ salt['pillar.get']('nginx:logdir', '/var/log/nginx') }}/{{ server_name }}{% if ssl %}-ssl{% endif %}.error.log;
+{%- endmacro %}
+
+{% macro server_definition(server_name, ssl=False, ipv6_address=none, letsencrypt=false, servernames=[]) -%}
+server {
+  server_name {{ server_name }}{%- for othername in servernames %}
+  {%- if othername != server_name %} {{ othername }}{% endif -%}
+  {% endfor -%};
+{% if ssl %}
+{%- if server_name == salt['grains.get']('nginx:default_servername') %}
+  listen 443 default_server ssl;
+  listen [::]:443 default_server ssl;
+{%- else %}
+  listen 443 ssl;
+  listen [::]:443;
+{%- endif %}
+{%- if letsencrypt %}
+
+  ssl_certificate /etc/letsencrypt/live/{{ server_name }}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/{{ server_name }}/privkey.pem;
+
+  # OCSP stapling
+  ssl_trusted_certificate /etc/letsencrypt/live/{{ server_name }}/chain.pem;
+{%- else %}
+
+  ssl_certificate {{ salt['pillar.get']('nginx:sslcertdir', '/etc/nginx/ssl/certs') }}/{{ server_name }}.crt.pem;
+  ssl_certificate_key {{ salt['pillar.get']('nginx:sslkeydir', '/etc/nginx/ssl/private') }}/{{ server_name }}.key.pem;
+
+  {%- if ca_certificate is defined and ca_certificate is not none %}
+  # OCSP stapling
+  ssl_trusted_certificate {{ ca_certificate }};
+  {%- endif %}
+{%- endif %}
+{%- else %}
+  listen 80;
+  listen [::]:80;
+{%- endif %}
+
+  {{ logfiles(server_name, ssl) }}
+{%- if not ssl %}
+{%- if letsencrypt %}
+
+  location /.well-known/acme-challenge {
+    root /srv/www/acme-challenge/{{ server_name }};
+  }
+{%- endif %}
+
+  location / {
+    return 301 https://$host$request_uri;
+  }
+{%- endif %}
+{%- endmacro %}

states/webserver/sslcert.macros.sls

@@ -9,7 +9,7 @@
     - bits: {{ salt['pillar.get']('nginx:keylength:' + domain_name, 2048) }}
     - require:
       - file: {{ nginx_ssl_keydir }}
-      - pkg: python-cryptography
+      - pkg: python3-cryptography
     - require_in:
       - file: /etc/nginx/sites-available/{{ domain_name }}
       - service: nginx
@@ -24,7 +24,7 @@
     - require:
       - file: {{ nginx_ssl_certdir }}
       - cmd: {{ certfile }}
-      - pkg: python-cryptography
+      - pkg: python3-cryptography
     - require_in:
       - file: /etc/nginx/sites-available/{{ domain_name }}
       - service: nginx