From cb62bd63e207909c989b34151ff8f49710b4ad7a Mon Sep 17 00:00:00 2001
From: Jan Dittberner <jan@dittberner.info>
Date: Sat, 22 Jul 2023 19:43:10 +0200
Subject: [PATCH] Add disk usage statistics
- add model CustomerPackageDiskUsage for hosting package disk usage
statistics
- add REST API endpoint for submittings statistics for disk, mysql and
pgsql usage
- add disk usage information to hosting package detail view
- add separate hosting package disk usage statistic view
---
gnuviechadmin/hostingpackages/admin.py | 14 +++
.../migrations/0007_add_disk_usage_table.py | 79 +++++++++++++++
gnuviechadmin/hostingpackages/models.py | 56 +++++++++++
gnuviechadmin/hostingpackages/serializers.py | 7 ++
.../customerhostingpackage_detail.html | 12 ++-
...omerhostingpackage_disk_usage_details.html | 91 ++++++++++++++++++
gnuviechadmin/hostingpackages/urls.py | 14 ++-
gnuviechadmin/hostingpackages/views.py | 95 +++++++++++++++++++
poetry.lock | 18 ++--
pyproject.toml | 9 +-
10 files changed, 379 insertions(+), 16 deletions(-)
create mode 100644 gnuviechadmin/hostingpackages/migrations/0007_add_disk_usage_table.py
create mode 100644 gnuviechadmin/hostingpackages/serializers.py
create mode 100644 gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_disk_usage_details.html
diff --git a/gnuviechadmin/hostingpackages/admin.py b/gnuviechadmin/hostingpackages/admin.py
index ae22212..716ebcd 100644
--- a/gnuviechadmin/hostingpackages/admin.py
+++ b/gnuviechadmin/hostingpackages/admin.py
@@ -12,6 +12,7 @@ from .models import (
CustomerHostingPackage,
CustomerHostingPackageDomain,
CustomerMailboxOption,
+ CustomerPackageDiskUsage,
CustomerUserDatabaseOption,
DiskSpaceOption,
HostingPackageTemplate,
@@ -95,6 +96,18 @@ class CustomerHostingPackageDomainInline(admin.TabularInline):
extra = 0
+class CustomerPackageDiskUsageInline(admin.TabularInline):
+ model = CustomerPackageDiskUsage
+ ordering = ["-used_kb", "source", "item"]
+ fields = ["source", "item", "used_kb"]
+ readonly_fields = ["source", "item", "used_kb"]
+ extra = 0
+ can_delete = False
+
+ def has_add_permission(self, request, obj):
+ return False
+
+
class CustomerHostingPackageAdmin(admin.ModelAdmin):
"""
This class implements the admin interface for
@@ -110,6 +123,7 @@ class CustomerHostingPackageAdmin(admin.ModelAdmin):
CustomerMailboxOptionInline,
CustomerUserDatabaseOptionInline,
CustomerHostingPackageDomainInline,
+ CustomerPackageDiskUsageInline,
]
list_display = ["name", "customer", "osuser"]
diff --git a/gnuviechadmin/hostingpackages/migrations/0007_add_disk_usage_table.py b/gnuviechadmin/hostingpackages/migrations/0007_add_disk_usage_table.py
new file mode 100644
index 0000000..7ba3753
--- /dev/null
+++ b/gnuviechadmin/hostingpackages/migrations/0007_add_disk_usage_table.py
@@ -0,0 +1,79 @@
+# Generated by Django 4.2.3 on 2023-07-22 17:31
+
+import django.utils.timezone
+import model_utils.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ replaces = [
+ ("hostingpackages", "0007_add_disk_usage_table"),
+ ("hostingpackages", "0008_add_default_for_used_kb_change_uniqueness"),
+ ]
+
+ dependencies = [
+ ("hostingpackages", "0006_auto_20150125_1510"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="CustomerPackageDiskUsage",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "created",
+ model_utils.fields.AutoCreatedField(
+ default=django.utils.timezone.now,
+ editable=False,
+ verbose_name="created",
+ ),
+ ),
+ (
+ "modified",
+ model_utils.fields.AutoLastModifiedField(
+ default=django.utils.timezone.now,
+ editable=False,
+ verbose_name="modified",
+ ),
+ ),
+ (
+ "source",
+ models.CharField(
+ choices=[
+ ("disk", "disk"),
+ ("mysql", "mysql"),
+ ("pgsql", "pgsql"),
+ ],
+ verbose_name="data source",
+ ),
+ ),
+ ("item", models.CharField(verbose_name="data item")),
+ (
+ "used_kb",
+ models.PositiveBigIntegerField(
+ default=0, verbose_name="space used in KiB"
+ ),
+ ),
+ (
+ "package",
+ models.ForeignKey(
+ help_text="The hosting package",
+ on_delete=django.db.models.deletion.CASCADE,
+ to="hostingpackages.customerhostingpackage",
+ verbose_name="hosting package",
+ ),
+ ),
+ ],
+ options={
+ "unique_together": {("package", "source", "item")},
+ },
+ ),
+ ]
diff --git a/gnuviechadmin/hostingpackages/models.py b/gnuviechadmin/hostingpackages/models.py
index 496083f..0f2a111 100644
--- a/gnuviechadmin/hostingpackages/models.py
+++ b/gnuviechadmin/hostingpackages/models.py
@@ -269,6 +269,31 @@ class CustomerHostingPackage(HostingPackageBase):
return DISK_SPACE_FACTORS[unit][min_unit] * diskspace
return DISK_SPACE_FACTORS[min_unit][unit] * diskspace
+ disk_space = property(get_disk_space)
+
+ def get_used_disk_space_sum(self, unit=None):
+ """
+ Get the used disk space of this hosting package from submitted disk space statistics.
+
+ :param unit: value from :py:data:`DISK_SPACE_UNITS` or :py:const:`None`
+ :return: disk space in unit or bytes (if parameter unit is :py:const:`None`)
+ :rtype: int
+
+ """
+ sum = 0
+ for usage in self.customerpackagediskusage_set.all():
+ sum += usage.size_in_bytes
+ if unit is None:
+ return sum
+ return DISK_SPACE_FACTORS[0][unit] * sum
+
+ used_disk_space_sum = property(get_used_disk_space_sum)
+
+ def get_space_level(self):
+ return self.used_disk_space_sum / self.disk_space * 100.0
+
+ space_level = property(get_space_level)
+
def get_package_space(self, unit=None):
"""
Get the total disk space reserved for this package without looking at
@@ -474,3 +499,34 @@ class CustomerMailboxOption(MailboxOptionBase, CustomerHostingPackageOption):
help_text=_("The mailbox option template that this mailbox option is based on"),
on_delete=models.CASCADE,
)
+
+
+class CustomerPackageDiskUsage(TimeStampedModel):
+ """
+ This class represents disk usage statistics for a customer hosting package.
+ """
+
+ package = models.ForeignKey(
+ CustomerHostingPackage,
+ verbose_name=_("hosting package"),
+ help_text=_("The hosting package"),
+ on_delete=models.CASCADE,
+ )
+ source = models.CharField(
+ verbose_name=_("data source"),
+ choices=(("disk", _("disk")), ("mysql", _("mysql")), ("pgsql", _("pgsql"))),
+ )
+ item = models.CharField(verbose_name=_("data item"))
+ used_kb = models.PositiveBigIntegerField(
+ verbose_name=_("space used in KiB"), default=0
+ )
+
+ class Meta:
+ unique_together = ("package", "source", "item")
+
+ def __str__(self):
+ return "%s %s = %d KiB" % (self.source, self.item, self.used_kb)
+
+ @property
+ def size_in_bytes(self):
+ return self.used_kb * 1024
diff --git a/gnuviechadmin/hostingpackages/serializers.py b/gnuviechadmin/hostingpackages/serializers.py
new file mode 100644
index 0000000..aeede05
--- /dev/null
+++ b/gnuviechadmin/hostingpackages/serializers.py
@@ -0,0 +1,7 @@
+from rest_framework import serializers
+
+from hostingpackages.models import CustomerPackageDiskUsage
+
+
+class DiskUsageSerializer(serializers.Serializer):
+ user = serializers.CharField()
diff --git a/gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_detail.html b/gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_detail.html
index c04be76..503a2a2 100644
--- a/gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_detail.html
+++ b/gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_detail.html
@@ -38,14 +38,16 @@
<dt>{% translate "Description" %}</dt>
<dd>{{ hostingpackage.description|default:"-" }}</dd>
<dt>{% translate "Disk space" %}</dt>
- {% with diskspace=hostingpackage.get_disk_space packagespace=hostingpackage.get_package_space %}
+ {% with used_space=hostingpackage.get_used_disk_space_sum|filesizeformat disk_space=hostingpackage.get_disk_space|filesizeformat package_space=hostingpackage.get_package_space|filesizeformat space_level=hostingpackage.space_level %}
<dd>
<span title="{% blocktranslate trimmed %}
- The reserved disk space for your hosting package is {{ diskspace }} bytes
-{% endblocktranslate %}">{{ diskspace|filesizeformat }}</span>
+ You use {{ used_space }} of the reserved disk space of {{ disk_space }} for your hosting package
+{% endblocktranslate %}" class="text-{% if space_level > 90.0 %}danger{% elif space_level > 80.0 %}warning{% else %}success{% endif %}">{% blocktranslate with space_level_percent=space_level|floatformat:1 trimmed%}
+ {{ used_space }} of {{ disk_space }} ({{ space_level_percent }}%)
+{% endblocktranslate %} <a title="{% translate "Disk usage details" %}" href="{% url "disk_usage_details" package=hostingpackage.id %}">{% translate "Details" %}</a></span>
<i class="bi-info-circle"
- title="{% blocktranslate with humanbytes=packagespace|filesizeformat trimmed %}
- The package contributes {{ humanbytes }} ({{ packagespace }} bytes) the difference comes from disk space options
+ title="{% blocktranslate trimmed %}
+ The package contributes {{ package_space }} the difference comes from disk space options
{% endblocktranslate %}"></i>
</dd>
{% endwith %}
diff --git a/gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_disk_usage_details.html b/gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_disk_usage_details.html
new file mode 100644
index 0000000..15f6111
--- /dev/null
+++ b/gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_disk_usage_details.html
@@ -0,0 +1,91 @@
+{% extends "hostingpackages/base.html" %}
+{% load i18n %}
+
+{% block title %}{{ block.super }} - {% spaceless %}
+ {% if user == customer %}
+ {% blocktranslate with package=hostingpackage.name trimmed %}
+ Disk usage details for your Hosting Package {{ package }}
+ {% endblocktranslate %}
+ {% else %}
+ {% blocktranslate with package=hostingpackage.name full_name=customer.get_full_name trimmed %}
+ Disk usage details for Hosting Package {{ package }} of {{ full_name }}
+ {% endblocktranslate %}
+ {% endif %}
+{% endspaceless %}{% endblock title %}
+
+{% block page_title %}{% blocktranslate with package=hostingpackage.name trimmed %}
+ Disk usage details for Hosting Package {{ package }}
+{% endblocktranslate %}{% endblock page_title %}
+
+{% block content %}
+ {% with used_space=hostingpackage.get_used_disk_space_sum|filesizeformat disk_space=hostingpackage.get_disk_space|filesizeformat package_space=hostingpackage.get_package_space|filesizeformat space_level=hostingpackage.space_level %}
+ <p>{% blocktranslate trimmed %}
+ You use {{ used_space }} of the reserved disk space of {{ disk_space }} for your hosting package.
+ {% endblocktranslate %}</p>
+ <p class="lead"><span
+ class="text-{% if space_level > 90.0 %}danger{% elif space_level > 80.0 %}warning{% else %}success{% endif %}">
+ {% blocktranslate with space_level_percent=space_level|floatformat:1 trimmed %}
+ {{ used_space }} of {{ disk_space }} ({{ space_level_percent }}%)
+ {% endblocktranslate %}</span>
+ <i class="bi-info-circle"
+ title="{% blocktranslate trimmed %}
+The package contributes {{ package_space }} the difference comes from disk space options
+{% endblocktranslate %}"></i>
+ </p>
+ <h2>{% trans "Breakdown by usage" %}</h2>
+ {% if disk_usage %}
+ <h3>{% trans "Regular file system usage" %}</h3>
+ <table class="table table-condensed">
+ <thead>
+ <tr>
+ <th>{% translate "Origin" %}</th>
+ <th class="text-end">{% translate "Used space" %}</th>
+ </tr>
+ </thead>
+ {% for line in disk_usage %}
+ <tr>
+ <td>{% if line.item == "web" %}{% translate "Website data" %}{% elif line.item == "mail" %}
+ {% translate "Mailboxes" %}{% elif line.item == "other" %}{% translate "Other" %}{% else %}
+ {{ line.item }}{% endif %}</td>
+ <td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
+ </tr>
+ {% endfor %}
+ </table>
+ {% endif %}
+ {% if mysql_usage %}
+ <h3>{% trans "MySQL/MariaDB database usage" %}</h3>
+ <table class="table table-condensed">
+ <thead>
+ <tr>
+ <th>{% translate "Database" %}</th>
+ <th class="text-end">{% translate "Used space" %}</th>
+ </tr>
+ </thead>
+ {% for line in mysql_usage %}
+ <tr>
+ <td>{{ line.item }}</td>
+ <td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
+ </tr>
+ {% endfor %}
+ </table>
+ {% endif %}
+ {% if pgsql_usage %}
+ <h3>{% trans "PostgreSQL database usage" %}</h3>
+ <table class="table table-condensed">
+ <thead>
+ <tr>
+ <th>{% translate "Database" %}</th>
+ <th class="text-end">{% translate "Used space" %}</th>
+ </tr>
+ </thead>
+ {% for line in pgsql_usage %}
+ <tr>
+ <td>{{ line.item }}</td>
+ <td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
+ </tr>
+ {% endfor %}
+ </table>
+ {% endif %}
+ {% endwith %}
+
+{% endblock content %}
diff --git a/gnuviechadmin/hostingpackages/urls.py b/gnuviechadmin/hostingpackages/urls.py
index fe3faa6..2734ff8 100644
--- a/gnuviechadmin/hostingpackages/urls.py
+++ b/gnuviechadmin/hostingpackages/urls.py
@@ -4,7 +4,7 @@ This module defines the URL patterns for hosting package related views.
"""
from __future__ import absolute_import
-from django.urls import re_path
+from django.urls import path, re_path
from .views import (
AddHostingOption,
@@ -12,7 +12,9 @@ from .views import (
CreateCustomerHostingPackage,
CreateHostingPackage,
CustomerHostingPackageDetails,
+ CustomerHostingPackageDiskUsageDetails,
HostingOptionChoices,
+ UploadCustomerPackageDiskUsage,
)
urlpatterns = [
@@ -42,4 +44,14 @@ urlpatterns = [
AddHostingOption.as_view(),
name="add_hosting_option",
),
+ path(
+ "<int:package>/disk-usage/",
+ CustomerHostingPackageDiskUsageDetails.as_view(),
+ name="disk_usage_details",
+ ),
+ path(
+ "upload-disk-usage/",
+ UploadCustomerPackageDiskUsage.as_view(),
+ name="upload_disk_usage",
+ ),
]
diff --git a/gnuviechadmin/hostingpackages/views.py b/gnuviechadmin/hostingpackages/views.py
index 37ff280..f071445 100644
--- a/gnuviechadmin/hostingpackages/views.py
+++ b/gnuviechadmin/hostingpackages/views.py
@@ -4,6 +4,9 @@ This module defines views related to hosting packages.
"""
from __future__ import absolute_import
+import http
+import logging
+
from django.conf import settings
from django.contrib import messages
from django.contrib.auth import get_user_model
@@ -13,6 +16,12 @@ from django.shortcuts import get_object_or_404, redirect
from django.utils.translation import gettext as _
from django.views.generic import DetailView, ListView
from django.views.generic.edit import CreateView, FormView
+
+import rest_framework.request
+from rest_framework.permissions import BasePermission
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
from gvacommon.viewmixins import StaffOrSelfLoginRequiredMixin
from .forms import (
@@ -24,10 +33,14 @@ from .forms import (
)
from .models import (
CustomerHostingPackage,
+ CustomerPackageDiskUsage,
DiskSpaceOption,
MailboxOption,
UserDatabaseOption,
)
+from .serializers import DiskUsageSerializer
+
+logger = logging.getLogger("gnuviechadmin.hostingpackages")
class CreateHostingPackage(PermissionRequiredMixin, CreateView):
@@ -259,3 +272,85 @@ class AddHostingOption(StaffUserRequiredMixin, FormView):
).format(option=option, package=hosting_package.name),
)
return redirect(hosting_package)
+
+
+class HasDiskUsageUploadPermission(BasePermission):
+ def has_permission(self, request, view):
+ return (
+ request.user.has_perm("hostingpackages.add_customerpackagediskusage")
+ and request.method == "POST"
+ )
+
+
+class UploadCustomerPackageDiskUsage(APIView):
+ permission_classes = [HasDiskUsageUploadPermission]
+ allowed_methods = ("POST",)
+ serializer = DiskUsageSerializer(many=True)
+
+ def post(self, request: rest_framework.request.Request, format=None):
+ if request.content_type != "application/json":
+ return Response("Unacceptable", status=http.HTTPStatus.BAD_REQUEST)
+ for row in request.data:
+ user = row["user"]
+ for key in row:
+ if key == "user":
+ continue
+ else:
+ for item, size in row[key].items():
+ try:
+ package = CustomerHostingPackage.objects.get(
+ osuser__username=user
+ )
+ (
+ metric,
+ created,
+ ) = CustomerPackageDiskUsage.objects.get_or_create(
+ package=package,
+ source=key,
+ item=item,
+ )
+ metric.used_kb = size
+ metric.save()
+ except CustomerHostingPackage.DoesNotExist:
+ logger.warning(
+ "hosting package for user %s does not exist", user
+ )
+
+ logger.info("usage date submitted by %s", request.user)
+
+ return Response("Accepted", status=http.HTTPStatus.ACCEPTED)
+
+
+class CustomerHostingPackageDiskUsageDetails(DetailView):
+ template_name_suffix = "_disk_usage_details"
+ model = CustomerHostingPackage
+ pk_url_kwarg = "package"
+ context_object_name = "hostingpackage"
+
+ def get_queryset(self, queryset=None):
+ return super().get_queryset().prefetch_related("customerpackagediskusage_set")
+
+ def get_context_data(self, **kwargs):
+ context_data = super().get_context_data(**kwargs)
+
+ disk_usage, mysql_usage, pgsql_usage = [], [], []
+
+ for usage in self.get_object().customerpackagediskusage_set.order_by(
+ "-used_kb"
+ ):
+ if usage.source == "disk":
+ disk_usage.append(usage)
+ elif usage.source == "mysql":
+ mysql_usage.append(usage)
+ elif usage.source == "pgsql":
+ pgsql_usage.append(usage)
+
+ context_data.update(
+ {
+ "disk_usage": disk_usage,
+ "mysql_usage": mysql_usage,
+ "pgsql_usage": pgsql_usage,
+ }
+ )
+
+ return context_data
diff --git a/poetry.lock b/poetry.lock
index 73c2bac..d87916c 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1051,20 +1051,20 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs
[[package]]
name = "isort"
-version = "4.3.21"
+version = "5.12.0"
description = "A Python utility / library to sort Python imports."
optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+python-versions = ">=3.8.0"
files = [
- {file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"},
- {file = "isort-4.3.21.tar.gz", hash = "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1"},
+ {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"},
+ {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"},
]
[package.extras]
-pipfile = ["pipreqs", "requirementslib"]
-pyproject = ["toml"]
-requirements = ["pip-api", "pipreqs"]
-xdg-home = ["appdirs (>=1.4.0)"]
+colors = ["colorama (>=0.4.3)"]
+pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"]
+plugins = ["setuptools"]
+requirements-deprecated-finder = ["pip-api", "pipreqs"]
[[package]]
name = "jinja2"
@@ -2153,4 +2153,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
[metadata]
lock-version = "2.0"
python-versions = "^3.8"
-content-hash = "25e51b747173bcb8fede3b14ee9c76c3bbce20bbf98aa906a08a74598471a4cc"
+content-hash = "8806d6bd5053ee7a90c76f20d25131e48bfd427d53d7474e851aeb6ee150e6b8"
diff --git a/pyproject.toml b/pyproject.toml
index 491719d..3dcb234 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,6 +25,7 @@ markdown = "^3.4.3"
django-filter = "^23.1"
crispy-bootstrap5 = "^0.7"
python-magic = "^0.4.27"
+isort = "^5.12.0"
[tool.poetry.group.dev.dependencies]
@@ -34,7 +35,6 @@ releases = "^2.0.0"
sphinxcontrib-blockdiag = "^3.0.0"
pylama = "^8.4.1"
black = {extras = ["d"], version = "^23.3.0"}
-isort = "<5"
[[tool.poetry.source]]
@@ -43,6 +43,13 @@ url = "https://pypi.gnuviech-server.de/simple"
priority = "explicit"
+[tool.isort]
+profile = "black"
+known_django = ["django","model_utils"]
+known_drf = ["rest_framework"]
+known_celery = ["celery"]
+sections = ["FUTURE", "STDLIB", "DJANGO", "DRF", "CELERY", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
+
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"