asynchronous refactoring

- don't execute celery tasks directly
- introduce optional parameters to fileserver tasks to allow chaining
- handle user/group/key create and delete tasks in new osusers.signals
  class
- adapt unit tests
- change TaskResults model to store the task signatures
- generalize the local settings' logging configuration
This commit is contained in:
Jan Dittberner 2015-10-12 00:23:31 +02:00
parent bcfea10e6f
commit d5bba7a22d
12 changed files with 290 additions and 170 deletions

View file

@ -9,7 +9,7 @@ from celery import shared_task
@shared_task
def setup_file_sftp_userdir(username):
def setup_file_sftp_userdir(username, *args, **kwargs):
"""
This task creates the home directory for an SFTP user if it does not exist
yet.
@ -23,7 +23,7 @@ def setup_file_sftp_userdir(username):
@shared_task
def delete_file_sftp_userdir(username):
def delete_file_sftp_userdir(username, *args, **kwargs):
"""
This task recursively deletes the home directory of an SFTP user if it
does not exist yet.
@ -37,7 +37,7 @@ def delete_file_sftp_userdir(username):
@shared_task
def setup_file_mail_userdir(username):
def setup_file_mail_userdir(username, *args, **kwargs):
"""
This task creates the mail base directory for a user if it does not exist
yet.
@ -52,7 +52,7 @@ def setup_file_mail_userdir(username):
@shared_task
def delete_file_mail_userdir(username):
def delete_file_mail_userdir(username, *args, **kwargs):
"""
This task recursively deletes the mail base directory for a user if it
does not exist yet.

View file

@ -50,7 +50,7 @@ def navigation(request):
request.path.endswith('/impressum/')
):
context['active_item'] = 'imprint'
else:
elif not viewmodule.startswith('django.contrib.admin'):
_LOGGER.debug(
'no special handling for view %s in module %s, fallback to '
'default active menu item %s',

View file

@ -261,6 +261,7 @@ ALLAUTH_APPS = (
LOCAL_APPS = (
'dashboard',
'taskresults',
'ldaptasks',
'mysqltasks',
'pgsqltasks',
'fileservertasks',

View file

@ -49,30 +49,13 @@ LOGGING['handlers'].update({
'formatter': 'simple',
}
})
LOGGING['loggers'].update({
'gnuviechadmin': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'dashboard': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'domains': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'gvacommon': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'gvawebcore': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'hostingpackages': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'managemails': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'osusers': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'taskresults': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'userdbs': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
'websites': {
'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,},
})
LOGGING['loggers'].update(dict(
[(key, {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True,})
for key in [
'dashboard', 'domains', 'fileservertasks', 'gvacommon',
'gvawebcore', 'hostingpackages', 'ldaptasks', 'managemails',
'mysqltasks', 'osusers', 'pgsqltasks', 'taskresults',
'userdbs', 'websites']]))
DEBUG_TOOLBAR_PATCH_SETTINGS = False

View file

@ -7,7 +7,6 @@ from __future__ import absolute_import
from celery import shared_task
@shared_task
def create_ldap_group(groupname, gid, descr):
"""

View file

@ -4,6 +4,7 @@ This module contains the :py:class:`django.apps.AppConfig` instance for the
"""
from __future__ import unicode_literals
from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _
@ -15,3 +16,6 @@ class OsusersAppConfig(AppConfig):
"""
name = 'osusers'
verbose_name = _('Operating System Users and Groups')
def ready(self):
import osusers.signals

View file

@ -10,9 +10,12 @@ import logging
import os
import six
from celery import group
from django.db import models, transaction
from django.conf import settings
from django.core.exceptions import ValidationError
from django.dispatch import Signal
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext as _
@ -24,28 +27,13 @@ from passlib.utils import generate_password
from taskresults.models import TaskResult
from ldaptasks.tasks import (
add_ldap_user_to_group,
create_ldap_group,
create_ldap_user,
delete_ldap_group,
delete_ldap_user,
remove_ldap_user_from_group,
set_ldap_user_password,
)
from fileservertasks.tasks import (
delete_file_mail_userdir,
delete_file_sftp_userdir,
set_file_ssh_authorized_keys,
setup_file_mail_userdir,
setup_file_sftp_userdir,
)
_LOGGER = logging.getLogger(__name__)
password_set = Signal(providing_args=['instance', 'password'])
CANNOT_USE_PRIMARY_GROUP_AS_ADDITIONAL = _(
"You can not use a user's primary group.")
@ -108,9 +96,6 @@ class Group(TimeStampedModel, models.Model):
"""
super(Group, self).save(*args, **kwargs)
dn = create_ldap_group.delay(
self.groupname, self.gid, self.descr).get()
_LOGGER.info("created LDAP group with dn %s", dn)
return self
@transaction.atomic
@ -124,10 +109,6 @@ class Group(TimeStampedModel, models.Model):
:py:meth:`django.db.Model.delete`
"""
TaskResult.objects.create_task_result(
delete_ldap_group.delay(self.groupname),
'delete_ldap_group'
)
super(Group, self).delete(*args, **kwargs)
@ -249,24 +230,12 @@ class User(TimeStampedModel, models.Model):
"""
if hasattr(self, 'shadow'):
self.shadow.set_password(password)
success = set_ldap_user_password.delay(
self.username, password).get()
if success:
_LOGGER.info(
"successfully set LDAP password for %s", self.username)
else:
_LOGGER.error(
"setting the LDAP password for %s failed", self.username)
return success
else:
self.shadow = Shadow.objects.create_shadow(
user=self, password=password
)
dn = create_ldap_user.delay(
self.username, self.uid, self.group.gid, self.gecos,
self.homedir, self.shell, password
).get()
_LOGGER.info("set LDAP password for %s", dn)
password_set.send(
sender=self.__class__, password=password, instance=self)
return True
def is_sftp_user(self):
@ -288,22 +257,6 @@ class User(TimeStampedModel, models.Model):
:rtype: :py:class:`osusers.models.User`
"""
dn = create_ldap_user.delay(
self.username, self.uid, self.group.gid, self.gecos,
self.homedir, self.shell, password=None).get()
TaskResult.objects.create_task_result(
setup_file_sftp_userdir.delay(self.username),
'setup_file_sftp_userdir'
)
TaskResult.objects.create_task_result(
setup_file_mail_userdir.delay(self.username),
'setup_file_mail_userdir'
)
_LOGGER.info(
"created user %(user)s with LDAP dn %(dn)s, scheduled home "
"directory and mail base directory creation.", {
'user': self, 'dn': dn,
})
return super(User, self).save(*args, **kwargs)
@transaction.atomic
@ -318,18 +271,6 @@ class User(TimeStampedModel, models.Model):
:py:meth:`django.db.Model.delete`
"""
TaskResult.objects.create_task_result(
delete_file_mail_userdir.delay(self.username),
'delete_file_mail_userdir'
)
TaskResult.objects.create_task_result(
delete_file_sftp_userdir.delay(self.username),
'delete_file_sftp_userdir'
)
for group in [ag.group for ag in self.additionalgroup_set.all()]:
remove_ldap_user_from_group.delay(
self.username, group.groupname).get()
delete_ldap_user.delay(self.username).get()
self.group.delete()
super(User, self).delete(*args, **kwargs)
@ -460,8 +401,6 @@ class AdditionalGroup(TimeStampedModel, models.Model):
:rtype: :py:class:`AdditionalGroup <osusers.models.AdditionalGroup>`
"""
add_ldap_user_to_group.delay(
self.user.username, self.group.groupname).get()
return super(AdditionalGroup, self).save(*args, **kwargs)
@transaction.atomic
@ -474,11 +413,6 @@ class AdditionalGroup(TimeStampedModel, models.Model):
:param kwargs: keyword arguments to be passed on to
:py:meth:`django.db.Model.delete`
"""
TaskResult.objects.create_task_result(
remove_ldap_user_from_group.delay(
self.user.username, self.group.groupname),
'remove_ldap_user_from_group'
)
super(AdditionalGroup, self).delete(*args, **kwargs)
@ -583,24 +517,3 @@ class SshPublicKey(TimeStampedModel):
return "{algorithm} {data} {comment}".format(
algorithm=self.algorithm, data=self.data, comment=self.comment
).strip()
def save(self, **kwargs):
key = super(SshPublicKey, self).save(**kwargs)
TaskResult.objects.create_task_result(
set_file_ssh_authorized_keys.delay(
self.user.username, [
str(key) for key in
SshPublicKey.objects.filter(user=self.user)]),
'set_file_ssh_authorized_keys'
)
return key
def delete(self, **kwargs):
super(SshPublicKey, self).delete(**kwargs)
TaskResult.objects.create_task_result(
set_file_ssh_authorized_keys.delay(
self.user.username, [
str(key) for key in
SshPublicKey.objects.filter(user=self.user)]),
'set_file_ssh_authorized_keys'
)

View file

@ -0,0 +1,160 @@
"""
This module contains the signal handlers of the :py:mod:`osusers` app.
"""
from __future__ import absolute_import, unicode_literals
import logging
from django.db.models.signals import (
m2m_changed,
post_delete,
post_save,
)
from django.dispatch import receiver
from celery import chain, group
from fileservertasks.tasks import (
delete_file_mail_userdir,
delete_file_sftp_userdir,
set_file_ssh_authorized_keys,
setup_file_mail_userdir,
setup_file_sftp_userdir,
)
from ldaptasks.tasks import (
add_ldap_user_to_group,
create_ldap_group,
create_ldap_user,
delete_ldap_group,
delete_ldap_user,
remove_ldap_user_from_group,
set_ldap_user_password,
)
from taskresults.models import TaskResult
from .models import (
AdditionalGroup,
Group,
SshPublicKey,
User,
password_set,
)
_LOGGER = logging.getLogger(__name__)
@receiver(password_set, sender=User)
def handle_user_password_set(sender, instance, password, **kwargs):
taskresult = TaskResult.objects.create_task_result(
'handle_user_password_set',
set_ldap_user_password.s(instance.username, password))
_LOGGER.info(
'LDAP password change has been requested in task %s',
taskresult.task_id)
#@receiver(post_save)
#def handle_post_save(sender, **kwargs):
# _LOGGER.debug(
# 'handling post_save signal for %s with args %s',
# sender, kwargs)
@receiver(post_save, sender=Group)
def handle_group_created(sender, instance, created, **kwargs):
if created:
taskresult = TaskResult.objects.create_task_result(
'handle_group_created',
create_ldap_group.s(
instance.groupname, instance.gid, instance.descr))
_LOGGER.info(
'LDAP group creation has been requested in task %s',
taskresult.task_id)
_LOGGER.debug(
'group %s has been %s', instance, created and "created" or "updated")
@receiver(post_save, sender=User)
def handle_user_created(sender, instance, created, **kwargs):
if created:
chain = create_ldap_user.s(
instance.username, instance.uid, instance.group.gid,
instance.gecos, instance.homedir, instance.shell, None
) | setup_file_sftp_userdir.s(instance.username
) | setup_file_mail_userdir.s(instance.username)
taskresult = TaskResult.objects.create_task_result(
'handle_user_created', chain)
_LOGGER.info(
'LDAP user creation has been requested in task %s',
taskresult.task_id)
_LOGGER.debug(
'user %s has been %s', instance, created and "created" or "updated")
@receiver(post_save, sender=AdditionalGroup)
def handle_user_added_to_group(sender, instance, created, **kwargs):
if created:
taskresult = TaskResult.objects.create_task_result(
'handle_user_added_to_group',
add_ldap_user_to_group.s(
instance.user.username, instance.group.groupname))
_LOGGER.info(
'Adding user to LDAP group has been requested in task %s',
taskresult.task_id)
@receiver(post_save, sender=SshPublicKey)
@receiver(post_delete, sender=SshPublicKey)
def handle_ssh_keys_changed(sender, instance, **kwargs):
sig = set_file_ssh_authorized_keys.s(
instance.user.username, [
str(key) for key in
SshPublicKey.objects.filter(user=instance.user)])
taskresult = TaskResult.objects.create_task_result(
'handle_ssh_keys_changed', sig)
_LOGGER.info(
'Change of SSH keys has been requested in task %s',
taskresult.task_id)
#@receiver(post_delete)
#def handle_post_delete(sender, **kwargs):
# _LOGGER.debug(
# 'handling post_delete signal for %s with args %s',
# sender, kwargs)
@receiver(post_delete, sender=Group)
def handle_group_deleted(sender, instance, **kwargs):
taskresult = TaskResult.objects.create_task_result(
'handle_group_deleted',
delete_ldap_group.s(instance.groupname))
_LOGGER.info(
'LDAP group deletion has been requested in task %s',
taskresult.task_id)
@receiver(post_delete, sender=User)
def handle_user_deleted(sender, instance, **kwargs):
chain = delete_file_mail_userdir.s(instance.username
) | delete_file_sftp_userdir.s(instance.username
) | delete_ldap_user.s(instance.username)
_LOGGER.debug('chain signature %s', chain)
taskresult = TaskResult.objects.create_task_result(
'handle_user_deleted', chain)
_LOGGER.info(
'LDAP user deletion has been requested in task %s',
taskresult.task_id)
@receiver(post_delete, sender=AdditionalGroup)
def handle_user_removed_from_group(sender, instance, **kwargs):
taskresult = TaskResult.objects.create_task_result(
'handle_user_removed_from_group',
remove_ldap_user_from_group.s(
instance.user.username, instance.group.groupname))
_LOGGER.info(
'Removing user from LDAP group has been requested in task %s',
taskresult.task_id)

View file

@ -128,8 +128,12 @@ class AdditionalGroupTest(TestCaseWithCeleryTasks):
addgroup = AdditionalGroup(user=self.user, group=group2)
addgroup.save()
taskres = TaskResult.objects.all()
self.assertTrue(len(taskres), 1)
self.assertEqual(taskres[0].task_name, 'setup_file_sftp_userdir')
self.assertTrue(len(taskres), 4)
creators = [r.creator for r in taskres]
for tcount, tcreator in [
(2, 'handle_group_created'), (1, 'handle_user_created'),
(1, 'handle_user_added_to_group')]:
self.assertEqual(creators.count(tcreator), tcount)
def test_delete(self):
group2 = Group.objects.create(groupname='test2', gid=1001)
@ -167,9 +171,9 @@ class GroupTest(TestCaseWithCeleryTasks):
self.assertEqual(len(Group.objects.all()), 1)
group.delete()
self.assertEqual(len(Group.objects.all()), 0)
self.assertEqual(len(TaskResult.objects.all()), 1)
self.assertEqual(len(TaskResult.objects.all()), 2)
tr = TaskResult.objects.first()
self.assertEqual(tr.task_name, 'delete_ldap_group')
self.assertEqual(tr.creator, 'handle_group_created')
class ShadowManagerTest(TestCaseWithCeleryTasks):
@ -275,10 +279,12 @@ class UserManagerTest(TestCaseWithCeleryTasks):
def test_create_user_tasks(self):
User.objects.create_user(customer=self.customer)
taskres = TaskResult.objects.all()
self.assertEqual(len(taskres), 2)
tasknames = [r.task_name for r in taskres]
self.assertEqual(tasknames.count('setup_file_sftp_userdir'), 1)
self.assertEqual(tasknames.count('setup_file_mail_userdir'), 1)
self.assertEqual(len(taskres), 3)
creators = [r.creator for r in taskres]
for creator in [
'handle_group_created', 'handle_user_created',
'handle_user_password_set']:
self.assertIn(creator, creators)
def test_create_user_second(self):
User.objects.create_user(customer=self.customer)
@ -351,24 +357,26 @@ class UserTest(TestCaseWithCeleryTasks):
def test_save(self):
user = User.objects.create_user(self.customer)
TaskResult.objects.all().delete()
user.save()
taskres = TaskResult.objects.all()
self.assertEqual(len(taskres), 2)
task_names = [r.task_name for r in taskres]
self.assertIn('setup_file_sftp_userdir', task_names)
self.assertIn('setup_file_mail_userdir', task_names)
self.assertEqual(len(taskres), 3)
creators = [r.creator for r in taskres]
for task in [
'handle_group_created', 'handle_user_created',
'handle_user_password_set']:
self.assertIn(task, creators)
def test_delete_only_user(self):
user = User.objects.create_user(self.customer)
TaskResult.objects.all().delete()
user.delete()
taskres = TaskResult.objects.all()
self.assertEqual(len(taskres), 3)
tasknames = [r.task_name for r in taskres]
self.assertEqual(tasknames.count('delete_file_mail_userdir'), 1)
self.assertEqual(tasknames.count('delete_file_sftp_userdir'), 1)
self.assertEqual(tasknames.count('delete_ldap_group'), 1)
self.assertEqual(len(taskres), 6)
creators = [r.creator for r in taskres]
for task in [
'handle_group_created', 'handle_user_created',
'handle_user_password_set', 'handle_user_deleted',
'handle_group_deleted', 'handle_user_deleted']:
self.assertIn(task, creators)
self.assertEqual(len(User.objects.all()), 0)
def test_delete_additional_groups(self):
@ -381,11 +389,12 @@ class UserTest(TestCaseWithCeleryTasks):
TaskResult.objects.all().delete()
user.delete()
taskres = TaskResult.objects.all()
self.assertEqual(len(taskres), 3)
tasknames = [t.task_name for t in taskres]
self.assertEqual(tasknames.count('delete_file_mail_userdir'), 1)
self.assertEqual(tasknames.count('delete_file_sftp_userdir'), 1)
self.assertEqual(tasknames.count('delete_ldap_group'), 1)
self.assertEqual(len(taskres), 5)
creators = [t.creator for t in taskres]
for tcount, tcreator in [
(2, 'handle_user_removed_from_group'), (2, 'handle_user_deleted'),
(1, 'handle_group_deleted')]:
self.assertEqual(creators.count(tcreator), tcount)
self.assertEqual(len(User.objects.all()), 0)
self.assertEqual(len(AdditionalGroup.objects.all()), 0)
@ -497,7 +506,7 @@ class SshPublicKeyTest(TestCaseWithCeleryTasks):
taskresults = TaskResult.objects.all()
self.assertEqual(len(taskresults), 1)
self.assertEqual(
taskresults[0].task_name, 'set_file_ssh_authorized_keys')
taskresults[0].creator, 'handle_ssh_keys_changed')
def test_call_tasks_on_delete(self):
key = SshPublicKey.objects.create_ssh_public_key(
@ -507,4 +516,4 @@ class SshPublicKeyTest(TestCaseWithCeleryTasks):
taskresults = TaskResult.objects.all()
self.assertEqual(len(taskresults), 1)
self.assertEqual(
taskresults[0].task_name, 'set_file_ssh_authorized_keys')
taskresults[0].creator, 'handle_ssh_keys_changed')

View file

@ -0,0 +1,36 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('taskresults', '0001_initial'),
]
operations = [
migrations.RemoveField(
model_name='taskresult',
name='task_name',
),
migrations.AddField(
model_name='taskresult',
name='creator',
field=models.TextField(default='migrated', verbose_name='Task creator'),
preserve_default=False,
),
migrations.AddField(
model_name='taskresult',
name='notes',
field=models.TextField(default='', verbose_name='Task notes'),
preserve_default=False,
),
migrations.AddField(
model_name='taskresult',
name='signature',
field=models.TextField(default='', verbose_name='Task signature'),
preserve_default=False,
),
]

View file

@ -12,15 +12,21 @@ from gnuviechadmin.celery import app
class TaskResultManager(models.Manager):
def create_task_result(self, asyncresult, name):
taskresult = self.create(task_id=asyncresult.id, task_name=name)
def create_task_result(self, creator, signature, notes=''):
sigstr = str(signature)
result = signature.apply_async()
taskresult = self.create(
task_id=result.task_id, creator=creator, signature=sigstr,
notes=notes)
return taskresult
@python_2_unicode_compatible
class TaskResult(models.Model):
task_id = models.CharField(_('Task id'), max_length=36)
task_name = models.CharField(_('Task name'), max_length=64)
signature = models.TextField(_('Task signature'))
creator = models.TextField(_('Task creator'))
notes = models.TextField(_('Task notes'))
result = models.TextField(_('Task result'))
finished = models.BooleanField(default=False)
state = models.CharField(_('Task state'), max_length=16)
@ -32,8 +38,8 @@ class TaskResult(models.Model):
verbose_name_plural = _('Task results')
def __str__(self):
return "{task_name} ({task_id}): {finished}".format(
task_name=self.task_name,
return "{creator} ({task_id}): {finished}".format(
creator=self.creator,
task_id=self.task_id,
finished=_('yes') if self.finished else _('no')
)
@ -41,7 +47,8 @@ class TaskResult(models.Model):
def fetch_result(self):
if not self.finished:
ar = app.AsyncResult(self.task_id)
res = ar.get(no_ack=True, timeout=1)
self.result = str(res)
self.state = ar.state
if ar.ready():
res = ar.get()
self.result = str(res)
self.finished = True

View file

@ -13,24 +13,30 @@ TEST_TASK_RESULT = '4ll y0ur b453 4r3 b3l0ng t0 u5'
class TaskResultTest(TestCase):
@patch('taskresults.models.app')
def test_update_taskstatus_unfinished(self, app):
mock = MagicMock(id=TEST_TASK_UUID, task_name=TEST_TASK_NAME)
mock.ready.return_value = False
tr = TaskResult.objects.create_task_result(mock, TEST_TASK_NAME)
resultmock = MagicMock(task_id=TEST_TASK_UUID)
resultmock.ready.return_value = False
mock = MagicMock()
mock.apply_async.return_value = resultmock
tr = TaskResult.objects.create_task_result(TEST_TASK_NAME, mock)
self.assertFalse(tr.finished)
mymock = app.AsyncResult(TEST_TASK_UUID)
mymock.state = 'SUCCESS'
mymock.get.return_value = TEST_RESULT
tr.fetch_result()
mymock.get.assert_called_with(no_ack=True, timeout=1)
mymock.get.assert_called_with()
self.assertTrue(tr.finished)
@patch('taskresults.models.app')
def test_update_taskstatus_finished(self, app):
mock = MagicMock(id=TEST_TASK_UUID, task_name=TEST_TASK_NAME)
mock.ready.return_value = True
resultmock = MagicMock(task_id=TEST_TASK_UUID)
resultmock.ready.return_value = True
resultmock.state = 'SUCCESS'
resultmock.result = TEST_RESULT
mock = MagicMock()
mock.apply_async.return_value = resultmock
mock.state = 'SUCCESS'
mock.result = TEST_RESULT
tr = TaskResult.objects.create_task_result(mock, TEST_TASK_NAME)
tr = TaskResult.objects.create_task_result(TEST_TASK_NAME, mock)
tr.fetch_result()
self.assertTrue(tr.finished)
mymock = app.AsyncResult(TEST_TASK_UUID)
@ -47,8 +53,10 @@ TEST_RESULT.ready.return_value = False
class TaskResultManagerTest(TestCase):
def test_create_task_result(self):
mock = MagicMock(id=TEST_TASK_UUID)
tr = TaskResult.objects.create_task_result(mock, TEST_TASK_NAME)
resultmock = MagicMock(task_id=TEST_TASK_UUID)
mock = MagicMock()
mock.apply_async.return_value = resultmock
tr = TaskResult.objects.create_task_result(TEST_TASK_NAME, mock)
self.assertIsInstance(tr, TaskResult)
self.assertEqual(tr.task_id, TEST_TASK_UUID)
self.assertEqual(tr.task_name, TEST_TASK_NAME)
self.assertEqual(tr.creator, TEST_TASK_NAME)