From 26aae0173d9a181f8a5ccb08cb6803f2937d8de0 Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Wed, 4 Mar 2020 16:53:50 +0100 Subject: [PATCH] Refactor fileservertasks into top level module This change moves the fileservertasks module to the top level to allow keeping the same task names when running in a Python 3 environment. --- docs/changelog.rst | 2 + docs/code.rst | 34 ++-- docs/conf.py | 148 +++++++++--------- gvafile/fileservertasks/__init__.py | 5 + .../{gvafile => fileservertasks}/celery.py | 6 +- .../{gvafile => fileservertasks}/settings.py | 23 ++- gvafile/fileservertasks/tasks.py | 2 +- gvafile/{gvafile => gvacommon}/__init__.py | 0 gvafile/gvacommon/celeryrouters.py | 7 + 9 files changed, 118 insertions(+), 109 deletions(-) rename gvafile/{gvafile => fileservertasks}/celery.py (57%) rename gvafile/{gvafile => fileservertasks}/settings.py (59%) rename gvafile/{gvafile => gvacommon}/__init__.py (100%) create mode 100644 gvafile/gvacommon/celeryrouters.py diff --git a/docs/changelog.rst b/docs/changelog.rst index b1fc443..b24a5e9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,8 @@ Changelog ========= +* :support:`-` move fileservertasks to top level to keep the task names when + using Python 3 * :support:`2` use Pipenv for dependency management * :release:`0.5.0 <2015-01-29>` diff --git a/docs/code.rst b/docs/code.rst index bf27672..84346b5 100644 --- a/docs/code.rst +++ b/docs/code.rst @@ -7,32 +7,26 @@ gvafile is implemented as a set of `Celery`_ tasks. .. _Celery: http://www.celeryproject.org/ -The project module :py:mod:`gvafile` -==================================== - -.. automodule:: gvafile - - -:py:mod:`celery ` ---------------------------------- - -.. automodule:: gvafile.celery - :members: - - -:py:mod:`settings ` -------------------------------------- - -.. automodule:: gvafile.settings - :members: - - :py:mod:`fileservertasks` module ================================ .. automodule:: fileservertasks +:py:mod:`celery ` +----------------------------------------- + +.. automodule:: fileservertasks.celery + :members: + + +:py:mod:`settings ` +--------------------------------------------- + +.. automodule:: fileservertasks.settings + :members: + + :py:mod:`tasks ` --------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index a9ea99c..b0fd370 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,164 +18,168 @@ import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath(os.path.join('..', 'gvafile'))) +sys.path.insert(0, os.path.abspath(os.path.join("..", "gvafile"))) -os.environ['GVAFILE_ALLOWED_HOSTS'] = 'localhost' -os.environ['GVAFILE_SERVER_EMAIL'] = 'root@localhost' -os.environ['GVAFILE_SFTP_DIRECTORY'] = '/home/www' -os.environ['GVAFILE_MAIL_DIRECTORY'] = '/home/mail' +os.environ["GVAFILE_BROKER_URL"] = "amqp://" +os.environ["GVAFILE_ALLOWED_HOSTS"] = "localhost" +os.environ["GVAFILE_MAIL_DIRECTORY"] = "/home/mail" +os.environ["GVAFILE_RESULTS_REDIS_URL"] = "redis://" +os.environ["GVAFILE_SERVER_EMAIL"] = "root@localhost" +os.environ["GVAFILE_SFTP_AUTHKEYS_DIRECTORY"] = "/srv/sftp/authorized_keys" +os.environ["GVAFILE_SFTP_DIRECTORY"] = "/home/www" # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['releases', 'sphinx.ext.autodoc', 'celery.contrib.sphinx'] +extensions = ["releases", "sphinx.ext.autodoc", "celery.contrib.sphinx"] + +# configuration for releases extension +releases_issue_uri = "https://git.dittberner.info/gnuviech/gvafile/issues/%s" +releases_release_uri = "https://git.dittberner.info/gnuviech/gvafile/src/tag/%s" # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -releases_issue_uri = 'https://dev.gnuviech-server.de/gvafile/ticket/%s' - -releases_release_uri = 'https://dev.gnuviech-server.de/gvafile/milestone/%s' +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'gvafile' -copyright = u'2014, 2015 Jan Dittberner' +project = u"gvafile" +copyright = u"2014-2020, Jan Dittberner" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.5.0' +from fileservertasks import __version__ as release + +# The short X.Y version. +version = ".".join(release.split(".")[:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'gvafiledoc' +htmlhelp_basename = "gvafiledoc" # -- Options for LaTeX output -------------------------------------------------- @@ -183,10 +187,8 @@ htmlhelp_basename = 'gvafiledoc' latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. #'preamble': '', } @@ -194,42 +196,38 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'gvafile.tex', u'gvafile Documentation', - u'Jan Dittberner', 'manual'), + ("index", "gvafile.tex", u"gvafile Documentation", u"Jan Dittberner", "manual") ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'gvafile', u'gvafile Documentation', - [u'Jan Dittberner'], 1) -] +man_pages = [("index", "gvafile", u"gvafile Documentation", [u"Jan Dittberner"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -238,16 +236,22 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'gvafile', u'gvafile Documentation', - u'Jan Dittberner', 'gvafile', 'GNUViech admin fileserver tools.', - 'Miscellaneous'), + ( + "index", + "gvafile", + u"gvafile Documentation", + u"Jan Dittberner", + "gvafile", + "GNUViech admin fileserver tools.", + "Miscellaneous", + ) ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' diff --git a/gvafile/fileservertasks/__init__.py b/gvafile/fileservertasks/__init__.py index cb2d3eb..96a388b 100644 --- a/gvafile/fileservertasks/__init__.py +++ b/gvafile/fileservertasks/__init__.py @@ -2,3 +2,8 @@ This module contains :py:mod:`fileservertasks.tasks`. """ +__version__ = "0.6.0" + +from fileservertasks.celery import app as celery_app + +__all__ = ('celery_app',) \ No newline at end of file diff --git a/gvafile/gvafile/celery.py b/gvafile/fileservertasks/celery.py similarity index 57% rename from gvafile/gvafile/celery.py rename to gvafile/fileservertasks/celery.py index b3b21a2..79b0051 100644 --- a/gvafile/gvafile/celery.py +++ b/gvafile/fileservertasks/celery.py @@ -9,7 +9,7 @@ from __future__ import absolute_import from celery import Celery #: The Celery application -app = Celery('gvafile') +app = Celery('fileservertasks') -app.config_from_object('gvafile.settings') -app.autodiscover_tasks(['fileservertasks'], force=True) +app.config_from_object('fileservertasks.settings') +app.autodiscover_tasks(['fileservertasks.tasks'], force=True) diff --git a/gvafile/gvafile/settings.py b/gvafile/fileservertasks/settings.py similarity index 59% rename from gvafile/gvafile/settings.py rename to gvafile/fileservertasks/settings.py index c099c38..b1816e0 100644 --- a/gvafile/gvafile/settings.py +++ b/gvafile/fileservertasks/settings.py @@ -25,23 +25,20 @@ def get_env_variable(setting): ########## CELERY CONFIGURATION -CELERY_TIMEZONE = 'Europe/Berlin' +CELERY_TIMEZONE = "Europe/Berlin" CELERY_ENABLE_UTC = True -CELERY_RESULT_BACKEND = get_env_variable('GVAFILE_RESULTS_REDIS_URL') +CELERY_RESULT_BACKEND = get_env_variable("GVAFILE_RESULTS_REDIS_URL") CELERY_RESULT_PERSISTENT = True CELERY_TASK_RESULT_EXPIRES = None -CELERY_ROUTES = ( - 'gvacommon.celeryrouters.GvaRouter', -) -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_RESULT_SERIALIZER = 'json' -BROKER_URL = get_env_variable('GVAFILE_BROKER_URL') +CELERY_ROUTES = ("gvacommon.celeryrouters.GvaRouter",) +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" +BROKER_URL = get_env_variable("GVAFILE_BROKER_URL") ########## END CELERY CONFIGURATION ########## GVAFILE CONFIGURATION -GVAFILE_SFTP_DIRECTORY = get_env_variable('GVAFILE_SFTP_DIRECTORY') -GVAFILE_MAIL_DIRECTORY = get_env_variable('GVAFILE_MAIL_DIRECTORY') -GVAFILE_SFTP_AUTHKEYS_DIRECTORY = get_env_variable( - 'GVAFILE_SFTP_AUTHKEYS_DIRECTORY') +GVAFILE_SFTP_DIRECTORY = get_env_variable("GVAFILE_SFTP_DIRECTORY") +GVAFILE_MAIL_DIRECTORY = get_env_variable("GVAFILE_MAIL_DIRECTORY") +GVAFILE_SFTP_AUTHKEYS_DIRECTORY = get_env_variable("GVAFILE_SFTP_AUTHKEYS_DIRECTORY") ########## END GVAFILE CONFIGURATION diff --git a/gvafile/fileservertasks/tasks.py b/gvafile/fileservertasks/tasks.py index 432163f..9ec26a8 100644 --- a/gvafile/fileservertasks/tasks.py +++ b/gvafile/fileservertasks/tasks.py @@ -11,7 +11,7 @@ import os import subprocess from tempfile import mkstemp -from gvafile import settings +from fileservertasks import settings from celery import shared_task from celery.utils.log import get_task_logger diff --git a/gvafile/gvafile/__init__.py b/gvafile/gvacommon/__init__.py similarity index 100% rename from gvafile/gvafile/__init__.py rename to gvafile/gvacommon/__init__.py diff --git a/gvafile/gvacommon/celeryrouters.py b/gvafile/gvacommon/celeryrouters.py new file mode 100644 index 0000000..c7f9db6 --- /dev/null +++ b/gvafile/gvacommon/celeryrouters.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +class GvaRouter(object): + def route_for_task(self, task, args=None, kwargs=None): + for route in ["ldap", "file", "mysql", "pgsql", "web"]: + if route in task: + return {"exchange": route, "exchange_type": "direct", "queue": route} + return None