Improve salt setup

This commit improves the salt setup of the Vagrant box: - Salt output is reduced to log level warning - Hosts entries are created for the internal IPs of all planned gva component VMs - .bashrc and a .bash_functions sourced from it are now managed for the vagrant user - the VM name has been changed to gva.local - recent salt versions do not depend on m2crypto anymore, therefore it is now installed before x509certificate functions are called - the rabbitmq_vhost for gva is now setup before any users are created because the previous implementation was broken with recent salt versions - the gnuviechadmin-locale-data-compile step has been simplified because Django 1.9's compilemessages takes care of recursive .mo file compilation - pillar data has been separated by role (especially queue permissions and credentials) - salt configuration is now unified with gvaldap
2016-01-29 18:34:40 +01:00 · 2016-01-29 18:34:40 +01:00 · 8396a0788d
commit 8396a0788d
parent 6fa4662bfd
35 changed files with 520 additions and 186 deletions
--- a/roots/gnuviechadmin/base.sls
+++ b/roots/gnuviechadmin/base.sls
@ -0,0 +1,98 @@
+{% from 'gnuviechadmin/vars.sls' import home, gva_component, gva_amqp_user, checkout, appdir, venv %}
+
+gva.local:
+  host.present:
+    - ip: 172.16.3.2
+    - names:
+      - mq
+      - gva.local
+
+gvaldap.local:
+  host.present:
+    - ip: 172.16.3.3
+
+gvafile.local:
+  host.present:
+    - ip: 172.16.3.4
+
+gvaweb.local:
+  host.present:
+    - ip: 172.16.3.5
+
+gvamysql.local:
+  host.present:
+    - ip: 172.16.3.6
+
+gvapgsql.local:
+  host.present:
+    - ip: 172.16.3.7
+
+gnuviechadmin-packages:
+  pkg.installed:
+    - pkgs:
+      - libyaml-dev
+      - python-virtualenv
+      - python-dev
+      - python-pip
+      - gettext
+
+{{ home }}/gvasettings.sh:
+  file.managed:
+    - user: vagrant
+    - group: vagrant
+    - mode: 0640
+    - source: salt://gnuviechadmin/{{ gva_component }}/settings.sh
+    - template: jinja
+    - context:
+        broker_url: {{ 'amqp://%s:%s@mq/%s' % (gva_amqp_user, salt['pillar.get']('gnuviechadmin:queues:users:%s:password' % gva_amqp_user), salt['pillar.get']('gnuviechadmin:queues:vhost')) }}
+
+gnuviechadmin-venv:
+  cmd.run:
+    - name: virtualenv {{ venv }}
+    - user: vagrant
+    - group: vagrant
+    - unless: test -f {{ venv }}/bin/pip
+
+gnuviechadmin-requires:
+  cmd.run:
+    - name: {{ venv }}/bin/pip install -U -r requirements/local.txt && touch {{ venv }}/lastinstall
+    - user: vagrant
+    - group: vagrant
+    - cwd: {{ checkout }}
+    - require:
+      - cmd: gnuviechadmin-venv
+      - pkg: gnuviechadmin-packages
+    - unless: test -e {{ venv }}/lastinstall && test {{ checkout }}/requirements/local.txt -ot {{ venv }}/lastinstall && test {{ checkout }}/requirements/base.txt -ot {{ venv }}/lastinstall
+
+gnuviechadmin-dbschema:
+  cmd.wait:
+    - name: . {{ home }}/gvasettings.sh ; unset LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER LC_TELEPHONE LC_TIME ; {{ venv }}/bin/python manage.py migrate --noinput
+    - user: vagrant
+    - group: vagrant
+    - cwd: {{ appdir }}
+    - watch:
+      - cmd: gnuviechadmin-requires
+      - file: {{ home }}/gvasettings.sh
+
+gnuviechadmin-locale-data-compile:
+  cmd.wait:
+    - name: . {{ home }}/gvasettings.sh ; {{ venv }}/bin/python {{ appdir }}/manage.py compilemessages
+    - user: vagrant
+    - group: vagrant
+    - cwd: {{ appdir }}
+    - require:
+      - pkg: gnuviechadmin-packages
+      - file: {{ home }}/gvasettings.sh
+      - cmd: gnuviechadmin-venv
+
+/home/vagrant/.bash_functions:
+  file.managed:
+    - user: vagrant
+    - group: vagrant
+    - mode: 0644
+    - source: salt://base/bash_functions
+    - template: jinja
+    - context:
+        home: {{ home }}
+        venv: {{ venv }}
+        appdir: {{ appdir }}
--- a/roots/gnuviechadmin/bash_functions
+++ b/roots/gnuviechadmin/bash_functions
@ -0,0 +1,25 @@
+#!/bin/bash
+
+function devenv
+{
+    . $HOME/gvasettings.sh
+    . $HOME/gva-venv/bin/activate
+    cd /vagrant/gnuviechadmin
+}
+
+function testenv
+{
+    devenv
+    export DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE%%.local}.test
+}
+
+function settitle
+{
+    if [ -n "$STY" ] ; then      # We are in a screen session
+        echo "Setting screen titles to $@"
+        printf "\033k%s\033\\" "$@"
+        screen -X eval "at \\# title $@" "shelltitle $@"
+    else
+        printf "\033]0;%s\007" "$@"
+    fi
+}
--- a/roots/gnuviechadmin/celery.sls
+++ b/roots/gnuviechadmin/celery.sls
@ -0,0 +1,13 @@
+{% from 'gnuviechadmin/vars.sls' import home, gva_component, venv, appdir %}
+
+{{ home }}/bin/run_celery.sh:
+  file.managed:
+    - user: vagrant
+    - group: vagrant
+    - mode: 0750
+    - source: salt://gnuviechadmin/{{ gva_component }}/run_celery.sh
+    - template: jinja
+    - context:
+        home: {{ home }}
+        virtualenv: {{ venv }}
+        appdir: {{ appdir }}
--- a/roots/gnuviechadmin/database.sls
+++ b/roots/gnuviechadmin/database.sls
@ -3,9 +3,9 @@ include:

 gnuviechadmin-database:
  postgres_user.present:
-    - name: {{ salt['pillar.get']('gnuviechadmin-database:owner:user') }}
+    - name: {{ salt['pillar.get']('gnuviechadmin:database:owner:user') }}
    - user: postgres
-    - password: {{ salt['pillar.get']('gnuviechadmin-database:owner:password') }}
+    - password: {{ salt['pillar.get']('gnuviechadmin:database:owner:password') }}
    - login: True
    - createdb: {% if salt['pillar.get']('gnuviechadmin:deploymenttype', 'production') == 'local' %}True
 {%- else %}False
@ -13,20 +13,20 @@ gnuviechadmin-database:
    - require:
      - service: postgresql
  postgres_database.present:
-    - name: {{ salt['pillar.get']('gnuviechadmin-database:database') }}
+    - name: {{ salt['pillar.get']('gnuviechadmin:database:name') }}
    - user: postgres
-    - owner: {{ salt['pillar.get']('gnuviechadmin-database:owner:user') }}
+    - owner: {{ salt['pillar.get']('gnuviechadmin:database:owner:user') }}
    - encoding: UTF8
    - template: template0
    - require:
      - service: postgresql
-      - postgres_user: {{ salt['pillar.get']('gnuviechadmin-database:owner:user') }}
+      - postgres_user: {{ salt['pillar.get']('gnuviechadmin:database:owner:user') }}

-{% for gnuviechadmin_db_role in salt['pillar.get']('gnuviechadmin-database:users') %}
+{% for gnuviechadmin_db_role in salt['pillar.get']('gnuviechadmin:database:users') %}
 gnuviechadmin-dbuser-{{ gnuviechadmin_db_role }}:
  postgres_user.present:
-    - name: {{ salt['pillar.get']('gnuviechadmin-database:users:' + gnuviechadmin_db_role + ':user') }}
-    - password: {{ salt['pillar.get']('gnuviechadmin-database:users:' + gnuviechadmin_db_role + ':password') }}
+    - name: {{ salt['pillar.get']('gnuviechadmin:database:users:%s:user' % gnuviechadmin_db_role) }}
+    - password: {{ salt['pillar.get']('gnuviechadmin:database:users:%s:password' % gnuviechadmin_db_role) }}
    - login: True
    - require:
      - service: postgresql
--- a/roots/gnuviechadmin/gva/gnuviechadmin.nginx
+++ b/roots/gnuviechadmin/gva/gnuviechadmin.nginx
--- a/roots/gnuviechadmin/gva/settings.sh
+++ b/roots/gnuviechadmin/gva/settings.sh
@ -3,11 +3,11 @@
 export DJANGO_SETTINGS_MODULE="gnuviechadmin.settings.{{ salt['pillar.get']('gnuviechadmin:deploymenttype', 'production') }}"
 export GVA_ADMIN_NAME="Jan Dittberner"
 export GVA_ADMIN_EMAIL="{{ salt['pillar.get']('gnuviechadmin:adminemail') }}"
-export GVA_PGSQL_DATABASE="{{ salt['pillar.get']('gnuviechadmin-database:database') }}"
-export GVA_PGSQL_USER="{{ salt['pillar.get']('gnuviechadmin-database:owner:user') }}"
-export GVA_PGSQL_PASSWORD="{{ salt['pillar.get']('gnuviechadmin-database:owner:password') }}"
-export GVA_PGSQL_HOSTNAME="{{ salt['pillar.get']('gnuviechadmin-database:hostname') }}"
-export GVA_PGSQL_PORT={{ salt['pillar.get']('gnuviechadmin-database:port') }}
+export GVA_PGSQL_DATABASE="{{ salt['pillar.get']('gnuviechadmin:database:name') }}"
+export GVA_PGSQL_USER="{{ salt['pillar.get']('gnuviechadmin:database:owner:user') }}"
+export GVA_PGSQL_PASSWORD="{{ salt['pillar.get']('gnuviechadmin:database:owner:password') }}"
+export GVA_PGSQL_HOSTNAME="{{ salt['pillar.get']('gnuviechadmin:database:host') }}"
+export GVA_PGSQL_PORT={{ salt['pillar.get']('gnuviechadmin:database:port') }}
 export GVA_DOMAIN_NAME="{{ salt['pillar.get']('gnuviechadmin:domainname') }}"
 export GVA_SITE_NAME="{{ salt['pillar.get']('gnuviechadmin:sitename') }}"
 export GVA_SITE_SECRET="{{ salt['grains.get_or_set_hash']('gnuviechadmin:SECRET_KEY', 50) }}"
--- a/roots/gnuviechadmin/gvaldap.sls
+++ b/roots/gnuviechadmin/gvaldap.sls
@ -0,0 +1,11 @@
+include:
+  - gnuviechadmin.base
+  - gnuviechadmin.celery
+
+gvaldap-packages:
+  pkg.installed:
+    - pkgs:
+      - libldap2-dev
+      - libsasl2-dev
+    - require_in:
+      - pkg: gnuviechadmin-packages
--- a/roots/gnuviechadmin/gvaldap/run_celery.sh
+++ b/roots/gnuviechadmin/gvaldap/run_celery.sh
@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -ex
+
+. {{ home }}/gvasettings.sh
+cd {{ appdir }}
+{{ virtualenv }}/bin/celery worker -A gvaldap -Q ldap --loglevel=INFO
--- a/roots/gnuviechadmin/gvaldap/settings.sh
+++ b/roots/gnuviechadmin/gvaldap/settings.sh
@ -0,0 +1,14 @@
+#!/bin/sh
+
+export DJANGO_SETTINGS_MODULE="gvaldap.settings.{{ salt['pillar.get']('gnuviechadmin:deploymenttype', 'production') }}"
+export GVALDAP_ADMIN_NAME="Jan Dittberner"
+export GVALDAP_ADMIN_EMAIL="{{ salt['pillar.get']('gnuviechadmin-gvaldap:admin_email') }}"
+export GVALDAP_LDAP_URL="{{ salt['pillar.get']('gnuviechadmin-gvaldap:ldap_url') }}"
+export GVALDAP_LDAP_USER="{{ salt['pillar.get']('gnuviechadmin-gvaldap:ldap_user') }}"
+export GVALDAP_LDAP_PASSWORD="{{ salt['pillar.get']('gnuviechadmin-gvaldap:ldap_password' ) }}"
+export GVALDAP_BASEDN_GROUP="{{ salt['pillar.get']('gnuviechadmin-gvaldap:basedn_group') }}"
+export GVALDAP_BASEDN_USER="{{ salt['pillar.get']('gnuviechadmin-gvaldap:basedn_user') }}"
+export GVALDAP_SECRETKEY="{{ salt['grains.get_or_set_hash']('gnuviechadmin-gvaldap:SECRET_KEY', 50) }}"
+export GVALDAP_BROKER_URL="{{ broker_url }}"
+export GVALDAP_ALLOWED_HOSTS="{{ salt['pillar.get']('gnuviechadmin-gvaldap:allowed_hosts') }}"
+export GVALDAP_SERVER_EMAIL="{{ salt['pillar.get']('gnuviechadmin-gvaldap:server_email') }}"
--- a/roots/gnuviechadmin/queues.sls
+++ b/roots/gnuviechadmin/queues.sls
@ -1,37 +1,30 @@
 include:
  - rabbitmq-server

-gnuviechadmin-queues:
-  rabbitmq_user.present:
-    - name: {{ salt['pillar.get']('gnuviechadmin-queues:owner:user') }}
-    - password: {{ salt['pillar.get']('gnuviechadmin-queues:owner:password') }}
-    - tags:
-      - administrator
-    - perms:
-      - {{ salt['pillar.get']('gnuviechadmin-queues:vhost') }}:
-        - '.*'
-        - '.*'
-        - '.*'
+gnuviechadmin-queue-vhost:
  rabbitmq_vhost.present:
-    - name: {{ salt['pillar.get']('gnuviechadmin-queues:vhost') }}
-    - owner: {{ salt['pillar.get']('gnuviechadmin-queues:owner:user') }}
-    - require:
-      - rabbitmq_user: {{ salt['pillar.get']('gnuviechadmin-queues:owner:user') }}
+    - name: {{ salt['pillar.get']('gnuviechadmin:queues:vhost') }}

-{% for user in salt['pillar.get']('gnuviechadmin-queues:users') %}
+{% for user in salt['pillar.get']('gnuviechadmin:queues:users') %}
 gnuviechadmin-queue-user-{{ user }}:
  rabbitmq_user.present:
    - name: {{ user }}
-    - password: {{ salt['pillar.get']('gnuviechadmin-queues:users:%s:password' % user) }}
-{% if salt['pillar.get']('gnuviechadmin-queues:users:%s:perms' % user) %}
+    - password: {{ salt['pillar.get']('gnuviechadmin:queues:users:%s:password' % user) }}
+{% if salt['pillar.get']('gnuviechadmin:queues:users:%s:perms' % user) %}
    - perms:
-{% for vhost, perms in salt['pillar.get']('gnuviechadmin-queues:users:%s:perms' % user).iteritems() %}
+{% for vhost, perms in salt['pillar.get']('gnuviechadmin:queues:users:%s:perms' % user).iteritems() %}
      - {{ vhost }}:
        - {{ perms[0] }}
        - {{ perms[1] }}
        - {{ perms[2] }}
 {% endfor %}
 {% endif %}
-    - require:
-      - rabbitmq_vhost: {{ salt['pillar.get']('gnuviechadmin-queues:vhost') }}
+{% if salt['pillar.get']('gnuviechadmin:queues:users:%s:tags' % user) %}
+    - tags:
+{% for tag in salt['pillar.get']('gnuviechadmin:queues:users:%s:tags' % user) %}
+      - {{ tag }}
+{% endfor %}
+{% endif %}
+    - require:
+      - rabbitmq_vhost: {{ salt['pillar.get']('gnuviechadmin:queues:vhost') }}
 {% endfor %}
--- a/roots/gnuviechadmin/vars.sls
+++ b/roots/gnuviechadmin/vars.sls
@ -0,0 +1,7 @@
+{% set home = '/home/vagrant' %}
+{% set venv = home + '/gva-venv' %}
+{% set checkout = '/vagrant' %}
+{% set gva_component = salt['pillar.get']('gnuviechadmin:component:name') %}
+{% set gva_amqp_user = salt['pillar.get']('gnuviechadmin:component:amqp_user') %}
+{% set python_module = salt['pillar.get']('gnuviechadmin:component:python_module', gva_component) %}
+{% set appdir = checkout + '/' + python_module %}
--- a/roots/gnuviechadmin/webinterface.sls
+++ b/roots/gnuviechadmin/webinterface.sls
@ -1,92 +1,27 @@
 include:
+  - gnuviechadmin.base
  - webserver

-mq:
-  host.present:
-    - ip: 127.0.0.1
-
-gnuviechadmin-packages:
+libpq-dev:
  pkg.installed:
-    - names:
-      - libpq-dev
-      - libyaml-dev
-      - python-virtualenv
-      - python-dev
-      - python-pip
-      - gettext
+    - require_in:
+      - pkg: gnuviechadmin-packages
+
+python-m2crypto:
+  pkg.installed:
+    - reload_modules: true

 {% import "webserver/sslcert.macros.sls" as sslcert %}

-{% set venv = salt['pillar.get']('gnuviechadmin:virtualenv') %}
-{% set checkout = '/vagrant' %}
-{% set home = '/home/vagrant' %}
-{% set appdir = checkout + '/gnuviechadmin' %}
 {% set domainname = salt['pillar.get']('gnuviechadmin:domainname') %}
 {{ sslcert.key_cert(domainname) }}

-{{ venv }}:
-  file.directory:
-    - user: vagrant
-    - group: vagrant
-    - require:
-      - cmd: gnuviechadmin-venv
-
-{{ home }}/gvasettings.sh:
-  file.managed:
-    - user: vagrant
-    - group: vagrant
-    - mode: 0640
-    - source: salt://gnuviechadmin/gvasettings.sh
-    - template: jinja
-    - context:
-        broker_url: amqp://{{ salt['pillar.get']('gnuviechadmin-queues:owner:user') }}:{{ salt['pillar.get']('gnuviechadmin-queues:owner:password') }}@mq/{{ salt['pillar.get']('gnuviechadmin-queues:vhost') }}
-
-gnuviechadmin-venv:
-  cmd.run:
-    - name: virtualenv {{ venv }}
-    - user: vagrant
-    - group: vagrant
-    - unless: test -f {{ venv }}/bin/pip
-
-gnuviechadmin-requires:
-  cmd.run:
-    - name: {{ venv }}/bin/pip install -U -r requirements/local.txt && touch {{ venv }}/lastinstall
-    - user: vagrant
-    - group: vagrant
-    - cwd: {{ checkout }}
-    - require:
-      - file: {{ venv }}
-      - pkg: python-dev
-      - pkg: libpq-dev
-    - unless: test -e {{ venv }}/lastinstall && test {{ checkout }}/requirements/local.txt -ot {{ venv }}/lastinstall && test {{ checkout }}/requirements/base.txt -ot {{ venv }}/lastinstall
-
-gnuviechadmin-dbschema:
-  cmd.wait:
-    - name: . {{ home }}/gvasettings.sh ; unset LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER LC_TELEPHONE LC_TIME ; {{ venv }}/bin/python manage.py migrate --noinput
-    - user: vagrant
-    - group: vagrant
-    - cwd: {{ appdir }}
-    - watch:
-      - cmd: gnuviechadmin-requires
-      - file: {{ home }}/gvasettings.sh
-
-gnuviechadmin-locale-data-compile:
-  cmd.wait:
-    - name: . {{ home }}/gvasettings.sh ; find {{ appdir }} -type d -name 'locale' | while read dir; do cd $(dirname "$dir") ; {{ venv }}/bin/python {{ appdir }}/manage.py compilemessages ; done
-    - user: vagrant
-    - group: vagrant
-    - cwd: {{ appdir }}
-    - require:
-      - pkg: gettext
-      - file: {{ home }}/gvasettings.sh
-      - file: {{ venv }}
-
 /etc/nginx/sites-available/{{ domainname }}:
  file.managed:
    - user: root
    - group: root
    - mode: 0640
-    - source: salt://gnuviechadmin/gnuviechadmin.nginx
+    - source: salt://gnuviechadmin/gva/gnuviechadmin.nginx
    - template: jinja
    - context:
        domainname: {{ domainname }}