gnuviech/gva
1
0
Fork 0
Code Issues 6 Pull requests Releases Wiki Activity

Add disk usage statistics

Browse source 
- add model CustomerPackageDiskUsage for hosting package disk usage
  statistics
- add REST API endpoint for submittings statistics for disk, mysql and
  pgsql usage
- add disk usage information to hosting package detail view
- add separate hosting package disk usage statistic view
This commit is contained in:
Jan Dittberner 2023-07-22 19:43:10 +02:00
parent affb49a971
commit cb62bd63e2
10 changed files with 379 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
gnuviechadmin/hostingpackages/admin.py
View file

@ -12,6 +12,7 @@ from .models import (
CustomerHostingPackage,
CustomerHostingPackageDomain,
CustomerMailboxOption,
CustomerPackageDiskUsage,
CustomerUserDatabaseOption,
DiskSpaceOption,
HostingPackageTemplate,
@ -95,6 +96,18 @@ class CustomerHostingPackageDomainInline(admin.TabularInline):
extra = 0
class CustomerPackageDiskUsageInline(admin.TabularInline):
model = CustomerPackageDiskUsage
ordering = ["-used_kb", "source", "item"]
fields = ["source", "item", "used_kb"]
readonly_fields = ["source", "item", "used_kb"]
extra = 0
can_delete = False
def has_add_permission(self, request, obj):
return False
class CustomerHostingPackageAdmin(admin.ModelAdmin):
"""
This class implements the admin interface for
@ -110,6 +123,7 @@ class CustomerHostingPackageAdmin(admin.ModelAdmin):
CustomerMailboxOptionInline,
CustomerUserDatabaseOptionInline,
CustomerHostingPackageDomainInline,
CustomerPackageDiskUsageInline,
]
list_display = ["name", "customer", "osuser"]

79
gnuviechadmin/hostingpackages/migrations/0007_add_disk_usage_table.py Normal file
View file

@ -0,0 +1,79 @@
# Generated by Django 4.2.3 on 2023-07-22 17:31
import django.utils.timezone
import model_utils.fields
from django.db import migrations, models
class Migration(migrations.Migration):
replaces = [
("hostingpackages", "0007_add_disk_usage_table"),
("hostingpackages", "0008_add_default_for_used_kb_change_uniqueness"),
]
dependencies = [
("hostingpackages", "0006_auto_20150125_1510"),
]
operations = [
migrations.CreateModel(
name="CustomerPackageDiskUsage",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
model_utils.fields.AutoCreatedField(
default=django.utils.timezone.now,
editable=False,
verbose_name="created",
),
),
(
"modified",
model_utils.fields.AutoLastModifiedField(
default=django.utils.timezone.now,
editable=False,
verbose_name="modified",
),
),
(
"source",
models.CharField(
choices=[
("disk", "disk"),
("mysql", "mysql"),
("pgsql", "pgsql"),
],
verbose_name="data source",
),
),
("item", models.CharField(verbose_name="data item")),
(
"used_kb",
models.PositiveBigIntegerField(
default=0, verbose_name="space used in KiB"
),
),
(
"package",
models.ForeignKey(
help_text="The hosting package",
on_delete=django.db.models.deletion.CASCADE,
to="hostingpackages.customerhostingpackage",
verbose_name="hosting package",
),
),
],
options={
"unique_together": {("package", "source", "item")},
},
),
]

56
gnuviechadmin/hostingpackages/models.py
View file

@ -269,6 +269,31 @@ class CustomerHostingPackage(HostingPackageBase):
return DISK_SPACE_FACTORS[unit][min_unit] * diskspace
return DISK_SPACE_FACTORS[min_unit][unit] * diskspace
disk_space = property(get_disk_space)
def get_used_disk_space_sum(self, unit=None):
"""
Get the used disk space of this hosting package from submitted disk space statistics.
:param unit: value from :py:data:`DISK_SPACE_UNITS` or :py:const:`None`
:return: disk space in unit or bytes (if parameter unit is :py:const:`None`)
:rtype: int
"""
sum = 0
for usage in self.customerpackagediskusage_set.all():
sum += usage.size_in_bytes
if unit is None:
return sum
return DISK_SPACE_FACTORS[0][unit] * sum
used_disk_space_sum = property(get_used_disk_space_sum)
def get_space_level(self):
return self.used_disk_space_sum / self.disk_space * 100.0
space_level = property(get_space_level)
def get_package_space(self, unit=None):
"""
Get the total disk space reserved for this package without looking at
@ -474,3 +499,34 @@ class CustomerMailboxOption(MailboxOptionBase, CustomerHostingPackageOption):
help_text=_("The mailbox option template that this mailbox option is based on"),
on_delete=models.CASCADE,
)
class CustomerPackageDiskUsage(TimeStampedModel):
"""
This class represents disk usage statistics for a customer hosting package.
"""
package = models.ForeignKey(
CustomerHostingPackage,
verbose_name=_("hosting package"),
help_text=_("The hosting package"),
on_delete=models.CASCADE,
)
source = models.CharField(
verbose_name=_("data source"),
choices=(("disk", _("disk")), ("mysql", _("mysql")), ("pgsql", _("pgsql"))),
)
item = models.CharField(verbose_name=_("data item"))
used_kb = models.PositiveBigIntegerField(
verbose_name=_("space used in KiB"), default=0
)
class Meta:
unique_together = ("package", "source", "item")
def __str__(self):
return "%s %s = %d KiB" % (self.source, self.item, self.used_kb)
@property
def size_in_bytes(self):
return self.used_kb * 1024

7
gnuviechadmin/hostingpackages/serializers.py Normal file
View file

@ -0,0 +1,7 @@
from rest_framework import serializers
from hostingpackages.models import CustomerPackageDiskUsage
class DiskUsageSerializer(serializers.Serializer):
user = serializers.CharField()

12
gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_detail.html
View file

@ -38,14 +38,16 @@
<dt>{% translate "Description" %}</dt>
<dd>{{ hostingpackage.description|default:"-" }}</dd>
<dt>{% translate "Disk space" %}</dt>
{% with diskspace=hostingpackage.get_disk_space packagespace=hostingpackage.get_package_space %}
{% with used_space=hostingpackage.get_used_disk_space_sum|filesizeformat disk_space=hostingpackage.get_disk_space|filesizeformat package_space=hostingpackage.get_package_space|filesizeformat space_level=hostingpackage.space_level %}
<dd>
<span title="{% blocktranslate trimmed %}
The reserved disk space for your hosting package is {{ diskspace }} bytes
{% endblocktranslate %}">{{ diskspace|filesizeformat }}</span>
You use {{ used_space }} of the reserved disk space of {{ disk_space }} for your hosting package
{% endblocktranslate %}" class="text-{% if space_level > 90.0 %}danger{% elif space_level > 80.0 %}warning{% else %}success{% endif %}">{% blocktranslate with space_level_percent=space_level|floatformat:1 trimmed%}
{{ used_space }} of {{ disk_space }} ({{ space_level_percent }}%)
{% endblocktranslate %} <a title="{% translate "Disk usage details" %}" href="{% url "disk_usage_details" package=hostingpackage.id %}">{% translate "Details" %}</a></span>
<i class="bi-info-circle"
title="{% blocktranslate with humanbytes=packagespace|filesizeformat trimmed %}
The package contributes {{ humanbytes }} ({{ packagespace }} bytes) the difference comes from disk space options
title="{% blocktranslate trimmed %}
The package contributes {{ package_space }} the difference comes from disk space options
{% endblocktranslate %}"></i>
</dd>
{% endwith %}

91
gnuviechadmin/hostingpackages/templates/hostingpackages/customerhostingpackage_disk_usage_details.html Normal file
View file

@ -0,0 +1,91 @@
{% extends "hostingpackages/base.html" %}
{% load i18n %}
{% block title %}{{ block.super }} - {% spaceless %}
{% if user == customer %}
{% blocktranslate with package=hostingpackage.name trimmed %}
Disk usage details for your Hosting Package {{ package }}
{% endblocktranslate %}
{% else %}
{% blocktranslate with package=hostingpackage.name full_name=customer.get_full_name trimmed %}
Disk usage details for Hosting Package {{ package }} of {{ full_name }}
{% endblocktranslate %}
{% endif %}
{% endspaceless %}{% endblock title %}
{% block page_title %}{% blocktranslate with package=hostingpackage.name trimmed %}
Disk usage details for Hosting Package {{ package }}
{% endblocktranslate %}{% endblock page_title %}
{% block content %}
{% with used_space=hostingpackage.get_used_disk_space_sum|filesizeformat disk_space=hostingpackage.get_disk_space|filesizeformat package_space=hostingpackage.get_package_space|filesizeformat space_level=hostingpackage.space_level %}
<p>{% blocktranslate trimmed %}
You use {{ used_space }} of the reserved disk space of {{ disk_space }} for your hosting package.
{% endblocktranslate %}</p>
<p class="lead"><span
class="text-{% if space_level > 90.0 %}danger{% elif space_level > 80.0 %}warning{% else %}success{% endif %}">
{% blocktranslate with space_level_percent=space_level|floatformat:1 trimmed %}
{{ used_space }} of {{ disk_space }} ({{ space_level_percent }}%)
{% endblocktranslate %}</span>
<i class="bi-info-circle"
title="{% blocktranslate trimmed %}
The package contributes {{ package_space }} the difference comes from disk space options
{% endblocktranslate %}"></i>
</p>
<h2>{% trans "Breakdown by usage" %}</h2>
{% if disk_usage %}
<h3>{% trans "Regular file system usage" %}</h3>
<table class="table table-condensed">
<thead>
<tr>
<th>{% translate "Origin" %}</th>
<th class="text-end">{% translate "Used space" %}</th>
</tr>
</thead>
{% for line in disk_usage %}
<tr>
<td>{% if line.item == "web" %}{% translate "Website data" %}{% elif line.item == "mail" %}
{% translate "Mailboxes" %}{% elif line.item == "other" %}{% translate "Other" %}{% else %}
{{ line.item }}{% endif %}</td>
<td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% if mysql_usage %}
<h3>{% trans "MySQL/MariaDB database usage" %}</h3>
<table class="table table-condensed">
<thead>
<tr>
<th>{% translate "Database" %}</th>
<th class="text-end">{% translate "Used space" %}</th>
</tr>
</thead>
{% for line in mysql_usage %}
<tr>
<td>{{ line.item }}</td>
<td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% if pgsql_usage %}
<h3>{% trans "PostgreSQL database usage" %}</h3>
<table class="table table-condensed">
<thead>
<tr>
<th>{% translate "Database" %}</th>
<th class="text-end">{% translate "Used space" %}</th>
</tr>
</thead>
{% for line in pgsql_usage %}
<tr>
<td>{{ line.item }}</td>
<td class="text-end">{{ line.size_in_bytes|filesizeformat }}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% endwith %}
{% endblock content %}

14
gnuviechadmin/hostingpackages/urls.py
View file

@ -4,7 +4,7 @@ This module defines the URL patterns for hosting package related views.
"""
from __future__ import absolute_import
from django.urls import re_path
from django.urls import path, re_path
from .views import (
AddHostingOption,
@ -12,7 +12,9 @@ from .views import (
CreateCustomerHostingPackage,
CreateHostingPackage,
CustomerHostingPackageDetails,
CustomerHostingPackageDiskUsageDetails,
HostingOptionChoices,
UploadCustomerPackageDiskUsage,
)
urlpatterns = [
@ -42,4 +44,14 @@ urlpatterns = [
AddHostingOption.as_view(),
name="add_hosting_option",
),
path(
"<int:package>/disk-usage/",
CustomerHostingPackageDiskUsageDetails.as_view(),
name="disk_usage_details",
),
path(
"upload-disk-usage/",
UploadCustomerPackageDiskUsage.as_view(),
name="upload_disk_usage",
),
]

95
gnuviechadmin/hostingpackages/views.py
View file

@ -4,6 +4,9 @@ This module defines views related to hosting packages.
"""
from __future__ import absolute_import
import http
import logging
from django.conf import settings
from django.contrib import messages
from django.contrib.auth import get_user_model
@ -13,6 +16,12 @@ from django.shortcuts import get_object_or_404, redirect
from django.utils.translation import gettext as _
from django.views.generic import DetailView, ListView
from django.views.generic.edit import CreateView, FormView
import rest_framework.request
from rest_framework.permissions import BasePermission
from rest_framework.response import Response
from rest_framework.views import APIView
from gvacommon.viewmixins import StaffOrSelfLoginRequiredMixin
from .forms import (
@ -24,10 +33,14 @@ from .forms import (
)
from .models import (
CustomerHostingPackage,
CustomerPackageDiskUsage,
DiskSpaceOption,
MailboxOption,
UserDatabaseOption,
)
from .serializers import DiskUsageSerializer
logger = logging.getLogger("gnuviechadmin.hostingpackages")
class CreateHostingPackage(PermissionRequiredMixin, CreateView):
@ -259,3 +272,85 @@ class AddHostingOption(StaffUserRequiredMixin, FormView):
).format(option=option, package=hosting_package.name),
)
return redirect(hosting_package)
class HasDiskUsageUploadPermission(BasePermission):
def has_permission(self, request, view):
return (
request.user.has_perm("hostingpackages.add_customerpackagediskusage")
and request.method == "POST"
)
class UploadCustomerPackageDiskUsage(APIView):
permission_classes = [HasDiskUsageUploadPermission]
allowed_methods = ("POST",)
serializer = DiskUsageSerializer(many=True)
def post(self, request: rest_framework.request.Request, format=None):
if request.content_type != "application/json":
return Response("Unacceptable", status=http.HTTPStatus.BAD_REQUEST)
for row in request.data:
user = row["user"]
for key in row:
if key == "user":
continue
else:
for item, size in row[key].items():
try:
package = CustomerHostingPackage.objects.get(
osuser__username=user
)
(
metric,
created,
) = CustomerPackageDiskUsage.objects.get_or_create(
package=package,
source=key,
item=item,
)
metric.used_kb = size
metric.save()
except CustomerHostingPackage.DoesNotExist:
logger.warning(
"hosting package for user %s does not exist", user
)
logger.info("usage date submitted by %s", request.user)
return Response("Accepted", status=http.HTTPStatus.ACCEPTED)
class CustomerHostingPackageDiskUsageDetails(DetailView):
template_name_suffix = "_disk_usage_details"
model = CustomerHostingPackage
pk_url_kwarg = "package"
context_object_name = "hostingpackage"
def get_queryset(self, queryset=None):
return super().get_queryset().prefetch_related("customerpackagediskusage_set")
def get_context_data(self, **kwargs):
context_data = super().get_context_data(**kwargs)
disk_usage, mysql_usage, pgsql_usage = [], [], []
for usage in self.get_object().customerpackagediskusage_set.order_by(
"-used_kb"
):
if usage.source == "disk":
disk_usage.append(usage)
elif usage.source == "mysql":
mysql_usage.append(usage)
elif usage.source == "pgsql":
pgsql_usage.append(usage)
context_data.update(
{
"disk_usage": disk_usage,
"mysql_usage": mysql_usage,
"pgsql_usage": pgsql_usage,
}
)
return context_data

18
poetry.lock generated
View file

@ -1051,20 +1051,20 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs
[[package]]
name = "isort"
version = "4.3.21"
version = "5.12.0"
description = "A Python utility / library to sort Python imports."
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
python-versions = ">=3.8.0"
files = [
{file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"},
{file = "isort-4.3.21.tar.gz", hash = "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1"},
{file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"},
{file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"},
]
[package.extras]
pipfile = ["pipreqs", "requirementslib"]
pyproject = ["toml"]
requirements = ["pip-api", "pipreqs"]
xdg-home = ["appdirs (>=1.4.0)"]
colors = ["colorama (>=0.4.3)"]
pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"]
plugins = ["setuptools"]
requirements-deprecated-finder = ["pip-api", "pipreqs"]
[[package]]
name = "jinja2"
@ -2153,4 +2153,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
[metadata]
lock-version = "2.0"
python-versions = "^3.8"
content-hash = "25e51b747173bcb8fede3b14ee9c76c3bbce20bbf98aa906a08a74598471a4cc"
content-hash = "8806d6bd5053ee7a90c76f20d25131e48bfd427d53d7474e851aeb6ee150e6b8"

9
pyproject.toml
View file

@ -25,6 +25,7 @@ markdown = "^3.4.3"
django-filter = "^23.1"
crispy-bootstrap5 = "^0.7"
python-magic = "^0.4.27"
isort = "^5.12.0"
[tool.poetry.group.dev.dependencies]
@ -34,7 +35,6 @@ releases = "^2.0.0"
sphinxcontrib-blockdiag = "^3.0.0"
pylama = "^8.4.1"
black = {extras = ["d"], version = "^23.3.0"}
isort = "<5"
[[tool.poetry.source]]
@ -43,6 +43,13 @@ url = "https://pypi.gnuviech-server.de/simple"
priority = "explicit"
[tool.isort]
profile = "black"
known_django = ["django","model_utils"]
known_drf = ["rest_framework"]
known_celery = ["celery"]
sections = ["FUTURE", "STDLIB", "DJANGO", "DRF", "CELERY", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"