- database versioning with migrate

- backend for domains
- settings for immutable things and config encapsulation


git-svn-id: file:///home/www/usr01/svn/gnuviechadmin/gnuviech.info/gnuviechadmin/trunk@229 a67ec6bc-e5d5-0310-a910-815c51eb3124

bin/gva

@@ -22,10 +22,14 @@
 
 import gnuviechadmin.cli.client
 import gnuviechadmin.cli.sysuser
+import gnuviechadmin.cli.domain
+import gnuviechadmin.cli.record
 import sys
 
 commands = [gnuviechadmin.cli.client.ClientCli,
-            gnuviechadmin.cli.sysuser.SysuserCli]
+            gnuviechadmin.cli.sysuser.SysuserCli,
+            gnuviechadmin.cli.domain.DomainCli,
+            gnuviechadmin.cli.record.RecordCli]
 
 def usage():
     print """%s <command> [commandargs]

data/dbrepo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://trac.erosson.com/migrate

data/dbrepo/manage.py

@@ -0,0 +1,4 @@
+#!/usr/bin/python
+from migrate.versioning.shell import main
+
+main(repository='data/dbrepo')

data/dbrepo/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Gnuviechadmin Schema Repository
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

data/dbrepo/versions/1/1.postgres.downgrade.sql

@@ -0,0 +1 @@
+DROP SCHEMA gva;

data/dbrepo/versions/1/1.postgres.upgrade.sql

@@ -0,0 +1 @@
+CREATE SCHEMA gva;

data/dbrepo/versions/2/2.py

@@ -0,0 +1,84 @@
+from sqlalchemy import *
+from migrate import *
+from gnuviechadmin.backend.settings import dbschema
+
+meta = BoundMetaData(migrate_engine)
+client = Table(
+    'client', meta,
+    Column('clientid', Integer, primary_key=True),
+    Column('title', String(10)),
+    Column('firstname', String(64), nullable=False),
+    Column('lastname', String(64), nullable=False),
+    Column('address1', String(64), nullable=False),
+    Column('address2', String(64)),
+    Column('zip', String(7), nullable=False),
+    Column('city', String(64), nullable=False),
+    Column('country', String(5), nullable=False),
+    Column('phone', String(32), nullable=False),
+    Column('mobile', String(32)),
+    Column('fax', String(32)),
+    Column('email', String(64), unique=True, nullable=False),
+    schema = dbschema
+    )
+sysuser = Table(
+    'sysuser', meta,
+    Column('sysuserid', Integer, primary_key=True),
+    Column('username', String(12), nullable=False, unique=True),
+    Column('usertype', Integer, nullable=False, default=0, index=True),
+    Column('home', String(128)),
+    Column('shell', Boolean, nullable=False, default=False),
+    Column('clearpass', String(64)),
+    Column('md5pass', String(34)),
+    Column('clientid', Integer, ForeignKey("client.clientid"),
+           nullable=False),
+    Column('sysuid', Integer, nullable=False, unique=True),
+    Column('lastchange', DateTime, default=func.now()),
+    schema = dbschema
+    )
+domain = Table(
+    'domain', meta,
+    Column('domainid', Integer, primary_key=True),
+    Column('name', String(255), nullable=False, unique=True),
+    Column('master', String(20)),
+    Column('last_check', Integer),
+    Column('type', String(6), nullable=False),
+    Column('notified_serial', Integer),
+    Column('sysuserid', Integer, ForeignKey("sysuser.sysuserid"),
+           nullable=False),
+    schema = dbschema
+    )
+record = Table(
+    'record', meta,
+    Column('recordid', Integer, primary_key=True),
+    Column('domainid', Integer, ForeignKey("domain.domainid"),
+           nullable=False),
+    Column('name', String(255)),
+    Column('type', String(6)),
+    Column('content', String(255)),
+    Column('ttl', Integer),
+    Column('prio', Integer),
+    Column('change_date', Integer),
+    schema = dbschema
+    )
+supermaster = Table(
+    'supermaster', meta,
+    Column('ip', String(25), nullable=False),
+    Column('nameserver', String(255), nullable=False),
+    Column('account', Integer, ForeignKey("sysuser.sysuserid"),
+           nullable=False),
+    schema = dbschema
+    )
+
+def upgrade():
+    client.create()
+    sysuser.create()
+    domain.create()
+    record.create()
+    supermaster.create()
+
+def downgrade():
+    supermaster.drop()
+    record.drop()
+    domain.drop()
+    sysuser.drop()
+    client.drop()

gnuviechadmin/backend/BackendEntityHandler.py

@@ -29,19 +29,19 @@ class BackendEntityHandler(object):
         self.verbose = verbose
 
     def create(self, **kwargs):
-        try:
-            entity = self.entityclass(self.verbose, **kwargs)
+#        try:
             sess = create_session()
-            sess.save(entity)
-            sess.flush()
+            entity = self.entityclass(self.verbose, **kwargs)
             try:
                 entity.create_hook()
+                sess.save(entity)
+                sess.flush()
             except:
                 sess.delete(entity)
                 sess.flush()
                 raise
-        except Exception, e:
-            raise CreationFailedError(self.entityclass.__name__, e)
+#        except Exception, e:
+#            raise CreationFailedError(self.entityclass.__name__, e)
 
     def fetchall(self):
         """Fetches all entities of the managed entity type."""

gnuviechadmin/backend/client.py

@@ -40,8 +40,8 @@ class Client(BackendEntity):
         self.address2 = None
         self.mobile = None
         self.fax = None
-        for item in kwargs.items():
-            self.__setattr__(item)
+        for (key, value) in kwargs.items():
+            self.__setattr__(key, value)
         self.validate()
 
     def create_hook(self):

gnuviechadmin/backend/domain.py

@@ -23,14 +23,17 @@ from sqlalchemy import *
 from tables import domain_table
 from gnuviechadmin.exceptions import *
 
-import record
+from record import Record
+import datetime
 from BackendEntity import *
 from BackendEntityHandler import *
+from settings import config
 
 class Domain(BackendEntity):
     """Entity class for DNS domains."""
 
     _shortkeys = ("domainid", "sysuserid", "name", "type")
+    _valid_domain_types = ("MASTER", "SLAVE")
 
     def __init__(self, verbose = False, **kwargs):
         BackendEntity.__init__(self, verbose)
@@ -38,10 +41,87 @@ class Domain(BackendEntity):
         self.sysuserid = None
         self.name = None
         self.type = None
+        self.master = None
+        self.ns1 = None
+        self.ns2 = None
+        self.mx = None
+        self.ipaddr = None
+        for (key, value) in kwargs.items():
+            self.__setattr__(key, value)
+        if not self.type:
+            self.type = self.getdefaultdomaintype()
+        if not self.ns1:
+            self.ns1 = config.get('domain', 'defaultns1')
+        if not self.ns2:
+            self.ns2 = config.get('domain', 'defaultns2')
+        if not self.mx:
+            self.mx = config.get('domain', 'defaultmx')
+        if not self.ipaddr:
+            self.ipaddr = config.get('domain', 'defaultip')
+        self.type = self.type.upper()
         self.validate()
 
+    def getdefaultdomaintype(self):
+        return self._valid_domain_types[0]
+
+    def validate(self):
+        BackendEntity.validate(self)
+        if not self.type in self._valid_domain_types:
+            raise ValidationFailedError(
+                self, "invalid domain type %s" % (self.type))
+        if self.type == 'SLAVE' and not self.master:
+            raise ValidationFailedError(
+                self, "you have to specify a master for slave domains.")
+        if self.type == 'MASTER':
+            if not self.ns1 or not self.ns2:
+                raise ValidationFailedError(
+                    self, "two nameservers must be specified.")
+            if not self.mx:
+                raise ValidationFailedError(
+                    self, "a primary mx host must be specified.")
+
+    def _getnewserial(self):
+        current = datetime.datetime.now()
+        return int("%04d%02d%02d01" % \
+                   (current.year, current.month, current.day))
+
+    def _getnewsoa(self):
+        return '%s %s %d %d %d %d %d' % \
+               (self.ns1,
+                config.get('domain', 'defaulthostmaster'),
+                self._getnewserial(),
+                config.getint('domain', 'defaultrefresh'),
+                config.getint('domain', 'defaultretry'),
+                config.getint('domain', 'defaultexpire'),
+                config.getint('domain', 'defaultminimumttl'))
+
+    def create_hook(self):
+        self.records.append(Record(
+            name = self.name, type = 'SOA',
+            content = self._getnewsoa(),
+            ttl = config.getint('domain', 'defaultttl')))
+        self.records.append(Record(
+            name = self.name, type = 'NS', content = self.ns1,
+            ttl = config.getint('domain', 'defaultttl')))
+        self.records.append(Record(
+            name = self.name, type = 'NS', content = self.ns2,
+            ttl = config.getint('domain', 'defaultttl')))
+        self.records.append(Record(
+            name  = self.name, type = 'MX', content = self.mx,
+            ttl = config.getint('domain', 'defaultttl'),
+            prio = config.getint('domain', 'defaultmxprio')))
+        self.records.append(Record(
+            name = self.name, type = 'A', content = self.ipaddr,
+            ttl = config.getint('domain', 'defaultttl')))
+        self.records.append(Record(
+            name = "www.%s" % (self.name), type = 'A', content = self.ipaddr,
+            ttl = config.getint('domain', 'defaultttl')))
+
+    def delete_hook(self):
+        pass
+
 domain_mapper = mapper(Domain, domain_table)
-domain_mapper.add_property("records", relation(record.Record))
+domain_mapper.add_property("records", relation(Record))
 
 class DomainHandler(BackendEntityHandler):
     """BackendEntityHandler for Domain entities."""

gnuviechadmin/backend/record.py

@@ -26,22 +26,25 @@ from gnuviechadmin.exceptions import *
 from BackendEntity import *
 from BackendEntityHandler import *
 
-class Record(BackendEntity):
+class Record(object):
     """Entity class for DNS domain records."""
-
-    _shortkeys = ("recordid", "domainid", "name", "type", "content")
-
-    def __init__(self, verbose = False, **kwargs):
-        BackendEntity.__init__(self, verbose)
-        self.recordid = None
-        self.domainid = None
-        self.name = None
-        self.type = None
-        self.content = None
-        self.ttl = None
-        self.prio = None
-        self.change_date = None
-        self.validate()
+    def __init__(self, **kwargs):
+        for (key, value) in kwargs.items():
+            self.__setattr__(key, value)
+
+    #_shortkeys = ("recordid", "domainid", "name", "type", "content")
+
+    #def __init__(self, verbose = False, **kwargs):
+    #    BackendEntity.__init__(self, verbose)
+    #    self.recordid = None
+    #    self.domainid = None
+    #    self.name = None
+    #    self.type = None
+    #    self.content = None
+    #    self.ttl = None
+    #    self.prio = None
+    #    self.change_date = None
+    #    self.validate()
 
 record_mapper = mapper(Record, record_table)
 

gnuviechadmin/backend/settings.py

@@ -0,0 +1,31 @@
+# -*- coding: UTF-8 -*-
+#
+# Copyright (C) 2007 by Jan Dittberner.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+#
+# Version: $Id$
+
+import ConfigParser, os
+
+# global settings which must not be user configurable
+required_version = 2
+dbschema = 'gva'
+
+# load user configuration
+config = ConfigParser.ConfigParser()
+config.readfp(open('gnuviechadmin/defaults.cfg'))
+config.read(['gnuviechadmin/gva.cfg', os.path.expanduser('~/.gva.cfg')])

gnuviechadmin/backend/sysuser.py

@@ -43,8 +43,8 @@ class Sysuser(BackendEntity):
         self.md5pass = None
         self.clientid = None
         self.sysuid = None
-        for key in kwargs.keys():
-            self.__setattr__(key, kwargs[key])
+        for (key, value) in kwargs.items():
+            self.__setattr__(key, value)
         if not self.username:
             self.username = self.getnextsysusername()
         if not self.usertype:

gnuviechadmin/backend/tables.py

@@ -20,11 +20,22 @@
 # Version: $Id$
 
 from sqlalchemy import *
-import ConfigParser, os
+import sys
+import migrate.versioning.api
+from settings import *
 
-config = ConfigParser.ConfigParser()
-config.readfp(open('gnuviechadmin/defaults.cfg'))
-config.read(['gnuviechadmin/gva.cfg', os.path.expanduser('~/.gva.cfg')])
+dbversion = migrate.versioning.api.db_version(
+    config.get('database', 'uri'),
+    config.get('database', 'repository'))
+if dbversion < required_version:
+    print("""Database version is %d but required version is %d, run
+
+ migrate upgrade %s %s
+
+to fix this.""" %
+          (dbversion, required_version, config.get('database', 'uri'),
+           config.get('database', 'repository')))
+    sys.exit(1)
 
 meta = BoundMetaData(config.get('database', 'uri'))
 client_table = Table(
@@ -42,10 +53,8 @@ client_table = Table(
     Column('mobile', String(32)),
     Column('fax', String(32)),
     Column('email', String(64), unique=True, nullable=False),
-    schema = config.get('database', 'schema')
+    schema = dbschema
     )
-client_table.create(checkfirst=True)
-
 sysuser_table = Table(
     'sysuser', meta,
     Column('sysuserid', Integer, primary_key=True),
@@ -55,13 +64,12 @@ sysuser_table = Table(
     Column('shell', Boolean, nullable=False, default=False),
     Column('clearpass', String(64)),
     Column('md5pass', String(34)),
-    Column('clientid', Integer, ForeignKey("client.clientid"), nullable=False),
+    Column('clientid', Integer, ForeignKey("client.clientid"),
+           nullable=False),
     Column('sysuid', Integer, nullable=False, unique=True),
     Column('lastchange', DateTime, default=func.now()),
-    schema = config.get('database', 'schema')
+    schema = dbschema
     )
-sysuser_table.create(checkfirst=True)
-
 domain_table = Table(
     'domain', meta,
     Column('domainid', Integer, primary_key=True),
@@ -72,10 +80,8 @@ domain_table = Table(
     Column('notified_serial', Integer),
     Column('sysuserid', Integer, ForeignKey("sysuser.sysuserid"),
            nullable=False),
-    schema = config.get('database', 'schema')
+    schema = dbschema
     )
-domain_table.create(checkfirst=True)
-
 record_table = Table(
     'record', meta,
     Column('recordid', Integer, primary_key=True),
@@ -87,16 +93,13 @@ record_table = Table(
     Column('ttl', Integer),
     Column('prio', Integer),
     Column('change_date', Integer),
-    schema = config.get('database', 'schema')
+    schema = dbschema
     )
-record_table.create(checkfirst=True)
-
 supermaster_table = Table(
     'supermaster', meta,
     Column('ip', String(25), nullable=False),
     Column('nameserver', String(255), nullable=False),
     Column('account', Integer, ForeignKey("sysuser.sysuserid"),
            nullable=False),
-    schema = config.get('database', 'schema')
+    schema = dbschema
     )
-supermaster_table.create(checkfirst=True)

gnuviechadmin/cli/__init__.py

@@ -24,4 +24,4 @@
 This package provides modules for the command line interface of the
 gnuviechadmin server administration suite."""
 
-__all__ = ["client", "sysuser"]
+__all__ = ["client", "sysuser", "domain", "record"]

gnuviechadmin/cli/client.py

@@ -22,7 +22,7 @@
 import CliCommand, sys
 
 class ClientCli(CliCommand.CliCommand):
-    """Command line interface command for client managament."""
+    """Command line interface command for client management."""
 
     name = "client"
     description = "manage clients"

gnuviechadmin/cli/domain.py

@@ -0,0 +1,66 @@
+# -*- coding: UTF-8 -*-
+#
+# Copyright (C) 2007 by Jan Dittberner.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+#
+# Version: $Id$
+
+import CliCommand, sys
+
+class DomainCli(CliCommand.CliCommand):
+    """Command line interface command for domain management."""
+
+    name = "domain"
+    description = "manage domains"
+    _optionmap = {
+        'create' : ("creates a new domain",
+                    [(["-n", "--name"], "name",
+                      "the domain name", True),
+                     (["-t", "--type"], "type",
+                      "domain type m for master or s for slave", False),
+                     (["-m", "--master"], "master",
+                      "master server for slave domains", False),
+                     (["-s", "--sysuserid"], "sysuserid",
+                      "system user id", True)]),
+        'list'   : ("lists existing domains",
+                    []),
+        'delete' : ("delete a domain",
+                    [(["-d", "--domainid"], "domainid",
+                      "the domain id", True)])}
+
+    def _execute(self, subcommand):
+        from gnuviechadmin.backend.domain import DomainHandler
+        from gnuviechadmin import exceptions
+        if subcommand == "create":
+            try:
+                mydomain = DomainHandler(self._verbose).create(
+                    **self._data)
+                if self._verbose:
+                    print mydomain
+            except exceptions.CreationFailedError, cfe:
+                self._usage()
+                print cfe
+                sys.exit(2)
+        elif subcommand == "list":
+            domains = DomainHandler(self._verbose).fetchall()
+            for domain in domains:
+                print domain
+        elif subcommand == "delete":
+            DomainHandler(self._verbose).delete(self._data["domainid"])
+
+    def __init__(self, argv):
+        CliCommand.CliCommand.__init__(self, argv)

gnuviechadmin/cli/record.py

@@ -0,0 +1,70 @@
+# -*- coding: UTF-8 -*-
+#
+# Copyright (C) 2007 by Jan Dittberner.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+#
+# Version: $Id$
+
+import CliCommand, sys
+
+class RecordCli(CliCommand.CliCommand):
+    """Command line interface command for DNS record management."""
+
+    name = "record"
+    description = "manage DNS records"
+    _optionmap = {
+        'create' : ("creates a new record",
+                    [(["-n", "--name"], "name",
+                      "the record name", True),
+                     (["-t", "--type"], "type",
+                      "record type", True),
+                     (["-c", "--content"], "content",
+                      "record content", True),
+                     (["-p", "--prio"], "prio",
+                      "MX record priority", False),
+                     (["--ttl"], "ttl",
+                      "Time to live", False),
+                     (["-d", "--domainid"], "domainid",
+                      "Domain id", True)]),
+        'list'   : ("lists existing records",
+                    []),
+        'delete' : ("delete a record",
+                    [(["-r", "--recordid"], "recordid",
+                      "the record id", True)])}
+
+    def _execute(self, subcommand):
+        from gnuviechadmin.backend.record import RecordHandler
+        from gnuviechadmin import exceptions
+        if subcommand == "create":
+            try:
+                myrecord = RecordHandler(self._verbose).create(
+                    **self._data)
+                if self._verbose:
+                    print myrecord
+            except exceptions.CreationFailedError, cfe:
+                self._usage()
+                print cfe
+                sys.exit(2)
+        elif subcommand == "list":
+            records = RecordHandler(self._verbose).fetchall()
+            for record in records:
+                print record
+        elif subcommand == "delete":
+            RecordHandler(self._verbose).delete(self._data["recordid"])
+
+    def __init__(self, argv):
+        CliCommand.CliCommand.__init__(self, argv)

gnuviechadmin/defaults.cfg

@@ -1,5 +1,3 @@
-# -*- python -*-
-#
 # Copyright (C) 2007 by Jan Dittberner.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -32,7 +30,7 @@
 # very usable for a real installation.
 #
 uri = sqlite:///:memory:
-schema = gva
+repository = /etc/gnuviechadmin/dbrepo
 
 [common]
 suwrapper = sudo

gnuviechadmin/exceptions.py

@@ -60,3 +60,16 @@ class DeleteFailedError(GnuviechadminError):
         if self.cause:
             msg += " The reason is %s." % (str(self.cause))
         return msg
+
+class ValidationFailedError(GnuviechadminError):
+    """This exception should be raised if the validation of a business
+    object failed."""
+    def __init__(self, instance, cause = None):
+        self.instance = instance
+        self.cause = cause
+
+    def __str__(self):
+        msg = "Validating %s failed." % (str(self.instance))
+        if self.cause:
+            msg += " The reason is %s." % (str(self.cause))
+        return msg