Add disk usage statistics

- add model CustomerPackageDiskUsage for hosting package disk usage
  statistics
- add REST API endpoint for submittings statistics for disk, mysql and
  pgsql usage
- add disk usage information to hosting package detail view
- add separate hosting package disk usage statistic view

gnuviechadmin/hostingpackages/admin.py

@@ -12,6 +12,7 @@ from .models import (
     CustomerHostingPackage,
     CustomerHostingPackageDomain,
     CustomerMailboxOption,
+    CustomerPackageDiskUsage,
     CustomerUserDatabaseOption,
     DiskSpaceOption,
     HostingPackageTemplate,
@@ -95,6 +96,18 @@ class CustomerHostingPackageDomainInline(admin.TabularInline):
     extra = 0
 
 
+class CustomerPackageDiskUsageInline(admin.TabularInline):
+    model = CustomerPackageDiskUsage
+    ordering = ["-used_kb", "source", "item"]
+    fields = ["source", "item", "used_kb"]
+    readonly_fields = ["source", "item", "used_kb"]
+    extra = 0
+    can_delete = False
+
+    def has_add_permission(self, request, obj):
+        return False
+
+
 class CustomerHostingPackageAdmin(admin.ModelAdmin):
     """
     This class implements the admin interface for
@@ -110,6 +123,7 @@ class CustomerHostingPackageAdmin(admin.ModelAdmin):
         CustomerMailboxOptionInline,
         CustomerUserDatabaseOptionInline,
         CustomerHostingPackageDomainInline,
+        CustomerPackageDiskUsageInline,
     ]
     list_display = ["name", "customer", "osuser"]
 

gnuviechadmin/hostingpackages/migrations/0007_add_disk_usage_table.py

@@ -0,0 +1,79 @@
+# Generated by Django 4.2.3 on 2023-07-22 17:31
+
+import django.utils.timezone
+import model_utils.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    replaces = [
+        ("hostingpackages", "0007_add_disk_usage_table"),
+        ("hostingpackages", "0008_add_default_for_used_kb_change_uniqueness"),
+    ]
+
+    dependencies = [
+        ("hostingpackages", "0006_auto_20150125_1510"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CustomerPackageDiskUsage",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    model_utils.fields.AutoCreatedField(
+                        default=django.utils.timezone.now,
+                        editable=False,
+                        verbose_name="created",
+                    ),
+                ),
+                (
+                    "modified",
+                    model_utils.fields.AutoLastModifiedField(
+                        default=django.utils.timezone.now,
+                        editable=False,
+                        verbose_name="modified",
+                    ),
+                ),
+                (
+                    "source",
+                    models.CharField(
+                        choices=[
+                            ("disk", "disk"),
+                            ("mysql", "mysql"),
+                            ("pgsql", "pgsql"),
+                        ],
+                        verbose_name="data source",
+                    ),
+                ),
+                ("item", models.CharField(verbose_name="data item")),
+                (
+                    "used_kb",
+                    models.PositiveBigIntegerField(
+                        default=0, verbose_name="space used in KiB"
+                    ),
+                ),
+                (
+                    "package",
+                    models.ForeignKey(
+                        help_text="The hosting package",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="hostingpackages.customerhostingpackage",
+                        verbose_name="hosting package",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("package", "source", "item")},
+            },
+        ),
+    ]

gnuviechadmin/hostingpackages/models.py

@@ -269,6 +269,31 @@ class CustomerHostingPackage(HostingPackageBase):
             return DISK_SPACE_FACTORS[unit][min_unit] * diskspace
         return DISK_SPACE_FACTORS[min_unit][unit] * diskspace
 
+    disk_space = property(get_disk_space)
+
+    def get_used_disk_space_sum(self, unit=None):
+        """
+        Get the used disk space of this hosting package from submitted disk space statistics.
+
+        :param unit: value from :py:data:`DISK_SPACE_UNITS` or :py:const:`None`
+        :return: disk space in unit or bytes (if parameter unit is :py:const:`None`)
+        :rtype: int
+
+        """
+        sum = 0
+        for usage in self.customerpackagediskusage_set.all():
+            sum += usage.size_in_bytes
+        if unit is None:
+            return sum
+        return DISK_SPACE_FACTORS[0][unit] * sum
+
+    used_disk_space_sum = property(get_used_disk_space_sum)
+
+    def get_space_level(self):
+        return self.used_disk_space_sum / self.disk_space * 100.0
+
+    space_level = property(get_space_level)
+
     def get_package_space(self, unit=None):
         """
         Get the total disk space reserved for this package without looking at
@@ -474,3 +499,34 @@ class CustomerMailboxOption(MailboxOptionBase, CustomerHostingPackageOption):
         help_text=_("The mailbox option template that this mailbox option is based on"),
         on_delete=models.CASCADE,
     )
+
+
+class CustomerPackageDiskUsage(TimeStampedModel):
+    """
+    This class represents disk usage statistics for a customer hosting package.
+    """
+
+    package = models.ForeignKey(
+        CustomerHostingPackage,
+        verbose_name=_("hosting package"),
+        help_text=_("The hosting package"),
+        on_delete=models.CASCADE,
+    )
+    source = models.CharField(
+        verbose_name=_("data source"),
+        choices=(("disk", _("disk")), ("mysql", _("mysql")), ("pgsql", _("pgsql"))),
+    )
+    item = models.CharField(verbose_name=_("data item"))
+    used_kb = models.PositiveBigIntegerField(
+        verbose_name=_("space used in KiB"), default=0
+    )
+
+    class Meta:
+        unique_together = ("package", "source", "item")
+
+    def __str__(self):
+        return "%s %s = %d KiB" % (self.source, self.item, self.used_kb)
+
+    @property
+    def size_in_bytes(self):
+        return self.used_kb * 1024

gnuviechadmin/hostingpackages/serializers.py

@@ -0,0 +1,7 @@
+from rest_framework import serializers
+
+from hostingpackages.models import CustomerPackageDiskUsage
+
+
+class DiskUsageSerializer(serializers.Serializer):
+    user = serializers.CharField()

gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_detail.html

@@ -38,14 +38,16 @@
                         <dt>{% translate "Description" %}</dt>
                         <dd>{{ hostingpackage.description|default:"-" }}</dd>
                         <dt>{% translate "Disk space" %}</dt>
-                        {% with diskspace=hostingpackage.get_disk_space packagespace=hostingpackage.get_package_space %}
+                        {% with used_space=hostingpackage.get_used_disk_space_sum|filesizeformat disk_space=hostingpackage.get_disk_space|filesizeformat package_space=hostingpackage.get_package_space|filesizeformat space_level=hostingpackage.space_level %}
                             <dd>
                                 <span title="{% blocktranslate trimmed %}
-                                The reserved disk space for your hosting package is {{ diskspace }} bytes
+                                You use {{ used_space }} of the reserved disk space of {{ disk_space }} for your hosting package
-{% endblocktranslate %}">{{ diskspace|filesizeformat }}</span>
+{% endblocktranslate %}" class="text-{% if space_level > 90.0 %}danger{% elif space_level > 80.0 %}warning{% else %}success{% endif %}">{% blocktranslate with space_level_percent=space_level|floatformat:1 trimmed%}
+                                    {{ used_space }} of {{ disk_space }} ({{ space_level_percent }}%)
+{% endblocktranslate %} <a title="{% translate "Disk usage details" %}" href="{% url "disk_usage_details" package=hostingpackage.id %}">{% translate "Details" %}</a></span>
                                 <i class="bi-info-circle"
-                                   title="{% blocktranslate with humanbytes=packagespace|filesizeformat trimmed %}
+                                   title="{% blocktranslate trimmed %}
-                                      The package contributes {{ humanbytes }} ({{ packagespace }} bytes) the difference comes from disk space options
+                                      The package contributes {{ package_space }} the difference comes from disk space options
 {% endblocktranslate %}"></i>
                             </dd>
                         {% endwith %}

gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_disk_usage_details.html

@@ -0,0 +1,91 @@
+{% extends "hostingpackages/base.html" %}
+{% load i18n %}
+
+{% block title %}{{ block.super }} - {% spaceless %}
+    {% if user == customer %}
+        {% blocktranslate with package=hostingpackage.name trimmed %}
+            Disk usage details for your Hosting Package {{ package }}
+        {% endblocktranslate %}
+    {% else %}
+        {% blocktranslate with package=hostingpackage.name full_name=customer.get_full_name trimmed %}
+            Disk usage details for Hosting Package {{ package }} of {{ full_name }}
+        {% endblocktranslate %}
+    {% endif %}
+{% endspaceless %}{% endblock title %}
+
+{% block page_title %}{% blocktranslate with package=hostingpackage.name trimmed %}
+    Disk usage details for Hosting Package {{ package }}
+{% endblocktranslate %}{% endblock page_title %}
+
+{% block content %}
+    {% with used_space=hostingpackage.get_used_disk_space_sum|filesizeformat disk_space=hostingpackage.get_disk_space|filesizeformat package_space=hostingpackage.get_package_space|filesizeformat space_level=hostingpackage.space_level %}
+        <p>{% blocktranslate trimmed %}
+            You use {{ used_space }} of the reserved disk space of {{ disk_space }} for your hosting package.
+        {% endblocktranslate %}</p>
+        <p class="lead"><span
+                class="text-{% if space_level > 90.0 %}danger{% elif space_level > 80.0 %}warning{% else %}success{% endif %}">
+            {% blocktranslate with space_level_percent=space_level|floatformat:1 trimmed %}
+                {{ used_space }} of {{ disk_space }} ({{ space_level_percent }}%)
+            {% endblocktranslate %}</span>
+            <i class="bi-info-circle"
+               title="{% blocktranslate trimmed %}
+The package contributes {{ package_space }} the difference comes from disk space options
+{% endblocktranslate %}"></i>
+        </p>
+        <h2>{% trans "Breakdown by usage" %}</h2>
+        {% if disk_usage %}
+            <h3>{% trans "Regular file system usage" %}</h3>
+            <table class="table table-condensed">
+                <thead>
+                <tr>
+                    <th>{% translate "Origin" %}</th>
+                    <th class="text-end">{% translate "Used space" %}</th>
+                </tr>
+                </thead>
+                {% for line in disk_usage %}
+                    <tr>
+                        <td>{% if line.item == "web" %}{% translate "Website data" %}{% elif line.item == "mail" %}
+                            {% translate "Mailboxes" %}{% elif line.item == "other" %}{% translate "Other" %}{% else %}
+                            {{ line.item }}{% endif %}</td>
+                        <td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
+                    </tr>
+                {% endfor %}
+            </table>
+        {% endif %}
+        {% if mysql_usage %}
+            <h3>{% trans "MySQL/MariaDB database usage" %}</h3>
+            <table class="table table-condensed">
+                <thead>
+                <tr>
+                    <th>{% translate "Database" %}</th>
+                    <th class="text-end">{% translate "Used space" %}</th>
+                </tr>
+                </thead>
+                {% for line in mysql_usage %}
+                    <tr>
+                        <td>{{ line.item }}</td>
+                        <td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
+                    </tr>
+                {% endfor %}
+            </table>
+        {% endif %}
+        {% if pgsql_usage %}
+            <h3>{% trans "PostgreSQL database usage" %}</h3>
+            <table class="table table-condensed">
+                <thead>
+                <tr>
+                    <th>{% translate "Database" %}</th>
+                    <th class="text-end">{% translate "Used space" %}</th>
+                </tr>
+                </thead>
+                {% for line in pgsql_usage %}
+                    <tr>
+                        <td>{{ line.item }}</td>
+                        <td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
+                    </tr>
+                {% endfor %}
+            </table>
+        {% endif %}
+    {% endwith %}
+
+{% endblock content %}

gnuviechadmin/hostingpackages/urls.py

@@ -4,7 +4,7 @@ This module defines the URL patterns for hosting package related views.
 """
 from __future__ import absolute_import
 
-from django.urls import re_path
+from django.urls import path, re_path
 
 from .views import (
     AddHostingOption,
@@ -12,7 +12,9 @@ from .views import (
     CreateCustomerHostingPackage,
     CreateHostingPackage,
     CustomerHostingPackageDetails,
+    CustomerHostingPackageDiskUsageDetails,
     HostingOptionChoices,
+    UploadCustomerPackageDiskUsage,
 )
 
 urlpatterns = [
@@ -42,4 +44,14 @@ urlpatterns = [
         AddHostingOption.as_view(),
         name="add_hosting_option",
     ),
+    path(
+        "<int:package>/disk-usage/",
+        CustomerHostingPackageDiskUsageDetails.as_view(),
+        name="disk_usage_details",
+    ),
+    path(
+        "upload-disk-usage/",
+        UploadCustomerPackageDiskUsage.as_view(),
+        name="upload_disk_usage",
+    ),
 ]

gnuviechadmin/hostingpackages/views.py

@@ -4,6 +4,9 @@ This module defines views related to hosting packages.
 """
 from __future__ import absolute_import
 
+import http
+import logging
+
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth import get_user_model
@@ -13,6 +16,12 @@ from django.shortcuts import get_object_or_404, redirect
 from django.utils.translation import gettext as _
 from django.views.generic import DetailView, ListView
 from django.views.generic.edit import CreateView, FormView
+
+import rest_framework.request
+from rest_framework.permissions import BasePermission
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
 from gvacommon.viewmixins import StaffOrSelfLoginRequiredMixin
 
 from .forms import (
@@ -24,10 +33,14 @@ from .forms import (
 )
 from .models import (
     CustomerHostingPackage,
+    CustomerPackageDiskUsage,
     DiskSpaceOption,
     MailboxOption,
     UserDatabaseOption,
 )
+from .serializers import DiskUsageSerializer
+
+logger = logging.getLogger("gnuviechadmin.hostingpackages")
 
 
 class CreateHostingPackage(PermissionRequiredMixin, CreateView):
@@ -259,3 +272,85 @@ class AddHostingOption(StaffUserRequiredMixin, FormView):
             ).format(option=option, package=hosting_package.name),
         )
         return redirect(hosting_package)
+
+
+class HasDiskUsageUploadPermission(BasePermission):
+    def has_permission(self, request, view):
+        return (
+            request.user.has_perm("hostingpackages.add_customerpackagediskusage")
+            and request.method == "POST"
+        )
+
+
+class UploadCustomerPackageDiskUsage(APIView):
+    permission_classes = [HasDiskUsageUploadPermission]
+    allowed_methods = ("POST",)
+    serializer = DiskUsageSerializer(many=True)
+
+    def post(self, request: rest_framework.request.Request, format=None):
+        if request.content_type != "application/json":
+            return Response("Unacceptable", status=http.HTTPStatus.BAD_REQUEST)
+        for row in request.data:
+            user = row["user"]
+            for key in row:
+                if key == "user":
+                    continue
+                else:
+                    for item, size in row[key].items():
+                        try:
+                            package = CustomerHostingPackage.objects.get(
+                                osuser__username=user
+                            )
+                            (
+                                metric,
+                                created,
+                            ) = CustomerPackageDiskUsage.objects.get_or_create(
+                                package=package,
+                                source=key,
+                                item=item,
+                            )
+                            metric.used_kb = size
+                            metric.save()
+                        except CustomerHostingPackage.DoesNotExist:
+                            logger.warning(
+                                "hosting package for user %s does not exist", user
+                            )
+
+        logger.info("usage date submitted by %s", request.user)
+
+        return Response("Accepted", status=http.HTTPStatus.ACCEPTED)
+
+
+class CustomerHostingPackageDiskUsageDetails(DetailView):
+    template_name_suffix = "_disk_usage_details"
+    model = CustomerHostingPackage
+    pk_url_kwarg = "package"
+    context_object_name = "hostingpackage"
+
+    def get_queryset(self, queryset=None):
+        return super().get_queryset().prefetch_related("customerpackagediskusage_set")
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+
+        disk_usage, mysql_usage, pgsql_usage = [], [], []
+
+        for usage in self.get_object().customerpackagediskusage_set.order_by(
+            "-used_kb"
+        ):
+            if usage.source == "disk":
+                disk_usage.append(usage)
+            elif usage.source == "mysql":
+                mysql_usage.append(usage)
+            elif usage.source == "pgsql":
+                pgsql_usage.append(usage)
+
+        context_data.update(
+            {
+                "disk_usage": disk_usage,
+                "mysql_usage": mysql_usage,
+                "pgsql_usage": pgsql_usage,
+            }
+        )
+
+        return context_data

poetry.lock

@@ -1051,20 +1051,20 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs
 
 [[package]]
 name = "isort"
-version = "4.3.21"
+version = "5.12.0"
 description = "A Python utility / library to sort Python imports."
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+python-versions = ">=3.8.0"
 files = [
-    {file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"},
+    {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"},
-    {file = "isort-4.3.21.tar.gz", hash = "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1"},
+    {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"},
 ]
 
 [package.extras]
-pipfile = ["pipreqs", "requirementslib"]
+colors = ["colorama (>=0.4.3)"]
-pyproject = ["toml"]
+pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"]
-requirements = ["pip-api", "pipreqs"]
+plugins = ["setuptools"]
-xdg-home = ["appdirs (>=1.4.0)"]
+requirements-deprecated-finder = ["pip-api", "pipreqs"]
 
 [[package]]
 name = "jinja2"
@@ -2153,4 +2153,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8"
-content-hash = "25e51b747173bcb8fede3b14ee9c76c3bbce20bbf98aa906a08a74598471a4cc"
+content-hash = "8806d6bd5053ee7a90c76f20d25131e48bfd427d53d7474e851aeb6ee150e6b8"

pyproject.toml

@@ -25,6 +25,7 @@ markdown = "^3.4.3"
 django-filter = "^23.1"
 crispy-bootstrap5 = "^0.7"
 python-magic = "^0.4.27"
+isort = "^5.12.0"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -34,7 +35,6 @@ releases = "^2.0.0"
 sphinxcontrib-blockdiag = "^3.0.0"
 pylama = "^8.4.1"
 black = {extras = ["d"], version = "^23.3.0"}
-isort = "<5"
 
 
 [[tool.poetry.source]]
@@ -43,6 +43,13 @@ url = "https://pypi.gnuviech-server.de/simple"
 priority = "explicit"
 
 
+[tool.isort]
+profile = "black"
+known_django = ["django","model_utils"]
+known_drf = ["rest_framework"]
+known_celery = ["celery"]
+sections = ["FUTURE", "STDLIB", "DJANGO", "DRF", "CELERY", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"