From 74dc8661b69408437a57a1c535d438dec5224162 Mon Sep 17 00:00:00 2001 From: "Karl O. Pinc" Date: Fri, 21 Jun 2024 16:06:54 -0500 Subject: [PATCH] Replace .ini config files with YAML config files --- .yamllint.yaml | 6 + MANIFEST.in | 3 + Makefile_pgwui.mk | 17 +- examples/etc/pgwui.yaml | 531 ++++++++++++++++++++++++ examples/misc/development.yaml | 547 +++++++++++++++++++++++++ pyproject.toml | 4 + setup.py | 1 + src/pgwui_server/exceptions.py | 13 - src/pgwui_server/pgwui_server.py | 125 ++---- tests/test_pgwui_server.py | 309 ++------------ tests/test_pgwui_server_integration.py | 67 +-- tox.ini | 2 + 12 files changed, 1200 insertions(+), 425 deletions(-) create mode 100644 .yamllint.yaml create mode 100644 examples/etc/pgwui.yaml create mode 100644 examples/misc/development.yaml diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..12159db --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,6 @@ +extends: default +ignore-from-file: .gitignore +rules: + document-start: disable + comments: + require-starting-space: false diff --git a/MANIFEST.in b/MANIFEST.in index 74fefd2..f0d494b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ recursive-include tests *.py # Include Makefile includes include *.mk include .coveragerc +include .yamllint.yaml # List all the examples, so we don't accidently include editor backups include examples/etc include examples/etc/systemd @@ -11,8 +12,10 @@ include examples/etc/nginx include examples/etc/nginx/sites-available include examples/etc/nginx/sites-available/mysite include examples/etc/pgwui.ini +include examples/etc/pgwui.yaml include examples/misc include examples/misc/development.ini +include examples/misc/development.yaml include LICENSE.txt include Makefile include src/pgwui_server/VERSION diff --git a/Makefile_pgwui.mk b/Makefile_pgwui.mk index 52d7348..97c646e 100644 --- a/Makefile_pgwui.mk +++ b/Makefile_pgwui.mk @@ -67,7 +67,7 @@ publish: check-manifest upload push ## run_tests Run regression tests .PHONY: run_tests -run_tests: devel/testenv +run_tests: devel/testenv dist if [ -x $(PYENV_BIN)/pyenv ] ; then \ (set -e ; \ export PYENV_ROOT=$(PYENV_INSTALLATION) ; \ @@ -182,6 +182,7 @@ update_testenv: devel/testenv # Development related targets +DEVEL_DEPS := setup.py pyproject.toml MANIFEST.in # Run linters .PHONY: run-linters @@ -190,12 +191,13 @@ run-linters: devel/pytest [ -e .yamllint.yaml ] && devel/pytest/bin/yamllint --strict . # Re-create development environment when build environment changes -devel: setup.py pyproject.toml MANIFEST.in +devel: $(DEVEL_DEPS) rm -rf devel ${TOX_STUFF} mkdir -p devel # virtualenv for package building -devel/buildenv: devel +devel/buildenv: $(DEVEL_DEPS) + mkdir -p devel [ -d devel/buildenv ] \ || ( ${VIRTUALENV} devel/buildenv ; \ devel/buildenv/bin/pip install --upgrade pip ; \ @@ -204,7 +206,8 @@ devel/buildenv: devel ) # virtualenv for development -devel/testenv: devel +devel/testenv: $(DEVEL_DEPS) + mkdir -p devel [ -d devel/testenv ] \ || ( ${VIRTUALENV} devel/testenv ; \ devel/testenv/bin/pip install --upgrade pip ; \ @@ -214,7 +217,8 @@ devel/testenv: devel ) # virtualenv for pytest and other code tests -devel/pytest: devel dist +devel/pytest: $(DEVEL_DEPS) + mkdir -p devel if [ ! -d devel/pytest ] ; then \ ( ${VIRTUALENV} devel/pytest ; \ devel/pytest/bin/pip install --upgrade pip ; \ @@ -234,7 +238,8 @@ devel/pytest: devel dist fi # virtualenv for pudb -devel/pudb: devel dist +devel/pudb: $(DEVEL_DEPS) + mkdir -p devel if [ ! -d devel/pudb ] ; then \ ( ${VIRTUALENV} devel/pudb ; \ devel/pudb/bin/pip install --upgrade pip ; \ diff --git a/examples/etc/pgwui.yaml b/examples/etc/pgwui.yaml new file mode 100644 index 0000000..eaab33f --- /dev/null +++ b/examples/etc/pgwui.yaml @@ -0,0 +1,531 @@ +### +# PGWUI_Server configuration +# +# Configuration changes require a server restart! +# (Unless you've done something not recommended to start the server.) +# +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### + +app: + + # + # PasteDeploy configuration + # + + # https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/paste.html#pastedeploy-entry-points + use: 'egg:pgwui_server' + + # + # PGWUI configuration + # + + pgwui: + # Postgres connection configuration. + + # Both pgwui.pg_host and pgwui.pg_port are optional; + # they default to the Unix PG socket and the default Postgres port. + # An empty string for host means use the unix socket. + pg_host: '' + pg_port: '5432' + + # The database to use by default. + # Not having the setting is the same as "", the empty string. + #pgwui.default_db: '' + + # The (PostgreSQL-named) encoding used for strings on the client-side. + # The list of PostgreSQL encodings can be found at: + # https://www.postgresql.org/docs/current/multibyte.html#MULTIBYTE-CHARSET-SUPPORTED + # This setting defaults to "auto", the special PostgreSQL value which + # sets the client_encoding based on the encoding of the client + # system's locale. + # Set this value to "", the empty string, to use the server's encoding + # as the client encoding. (client_encoding: '') (or just omit the + # declaration) + #client_encoding: 'auto' + + + # How to link to the site's home page. Useful when the home page is + # not the PGWUI menu. + # + # This setting is a mapping (a dict). The keys depend on the "type" + # used. + # + # type: + # URL An URL. This is the default. + # file A file containing HTML, read from the file system. + # asset A Pyramid asset specification. Results in an URL + # which references a file which is part of a PGWUI + # component, or some other "static" file included in a Pyramid + # application. + # route A Pyramid route name. Results in an URL + # which references a page generated by a Pyramid application. + # "route_prefix" is not applied. + # + # + # When "type" is "URL", there are the following keys: + # + # source: (required) + # + # The default is ``/``, when there is no home_page setting. + # Which produces an URL with no "path". + # + # * A URI path beginning with ``/``. E.g.: '/home' + # + # * An URL without a protocol, so an URL beginning with ``//`` and + # followed by a domain. E.g.: //www.example.com The URL + # delivered to the browser contains the protocol used in the request. + # + # * An URL with a protocol. E.g.: https://www.example.com + # + # + # When "type" is "file", there are the following keys + # + # source: (required) + # A fully-qualified file system path, so a path beginning with + # a ``/``. E.g. /var/www/html/index.html + # Served with a content encoding of ``text/html``. + # + # url_path: (required) + # The "path" component of the URL used to retrieve the file. + # Must begin with a ``/``. "route_prefix" is not applied. + # + # When type is "asset", there are the following keys: + # + # source: (required) + # + # * A `Pyramid`_ `asset specification`_. It must reference a + # `static asset`_, a file included in a `Pyramid`_ application. + # A file containing a page of HTML. + # + # + # When type is "route", there are the following keys: + # + # source: (required) + # + # * A `Pyramid`_ `route name`_. Used to reach a page generated + # by a `Pyramid`_ application which uses `URL dispatch`_. + # + # + # This is the default: + #home_page: + # type: 'URL' + # source: '/' + + # How to link to a menu of PGWUI components. An alternative to using + # the PGWUI_Menu component. + # Configured as pgwui.home_page is configured, above. + # The default is to have no menu. + # + # The following example uses the URL without a path (e.g., + # http://www.example.com/) as the menu page: + # + # menu_page: + # type: 'URL' + # source: '/' + # + # The "menu_page" mapping overrides what is provided by the + # PGWUI_Menu component. + + # Whether to auto-discover the pgwui component modules. (boolean, optional) + # When false pgwui component names must be listed in + # pyramid.includes: ... + #autoconfigure: true + + # What PGWUI components and other pyramid modules to use. + # (Required when pgwui.autoconfigure is false, or you want the + # debug toolbar.) + # sample that includes the PGUI logout and upload components, + # in the block-style list format rather than the inline + # list format as appears in the Pyramid configuration section, below: + #pyramid.includes + # - 'pgwui_logout' + # - 'pgwui_upload' + + # Whether or not to change the db content. (boolean, required) + dry_run: false + + # routing + + # Routes are what call up specific pages. They are usually the + # part of the URL which comes after the http://example.com. + # + # All routes should probably begin with a "/" character but + # if they do not a "/" will be automatically prepended. + # + # A full URL may be given as a route. The is useful for redirecting + # to other sites. + # + # For more information on route syntax see: + # https://docs.pylonsproject.org/projects/pyramid/en/master/narr/urldispatch.html#route-pattern-syntax + + # A prefix for all routes. (optional) If the prefix is "/a/b/c" then + # all URLs will begin with something like: http://example.com/a/b/c + # The default is no prefix. + #route_prefix: '' + + # Overriding routes of specific PGWUI components + + # The syntax is name: "route", one per line. The "name" is the + # name of the PGWUI component. "route" is the route to use to access + # the component. So to access the logout page at + # http://example.com/logmeout the line would be: + # pgwui_logout: '/logmeout' + # + # Overriding routes is optional. + # + # The default for some PGWUI components are: + #routes: + # pgwui_copy: '/copy' + # pgwui_logout: '/logout' + # pgwui_upload: '/upload' + + # Overriding assets + # + # The visual presentation of a PGWUI component is controlled by its assets. + # Overriding assets allows for extensive customization of presentation. + # + # A Pyramid asset is any file included in a Pyramid application that is + # not a Python source code file. So assets are image files, Mako template + # files used to render pages, CSS files, JavaScript files, etc. + # + # See: + # https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/narr/assets.html + # + # By default no assets are overridden. Override assets with: + # + # override_assets: + # asset: 'new' + # + # 'asset' is the asset to override, a Pyramid asset specification + # + # 'new' is the new value either a Pyramid asset specification + # or a file system path + # + # Example, altering the menu presented by PGWUI_Menu: + # + # override_assets: + # 'pgwui_menu:templates/menu.mak': '/tmp/mymenu.mak' + + # Settings validation + + # Whether or not to validate the Beaker session.secret + # setting. (boolean, optional) + # session.secret must be valid to detect Cross-Site Request Forgery (CSRF) + # vulnerabilties. Validation is on by default. + #validate_hmac: true + + + # PGWUI Component Settings + + # Menu presentation + + # The PGWUI_Menu component automaticaly constructs a menu for the + # PGWUI components in use. The display value of the menu items can + # be overridden using a "menu_label" setting for each component. + # CAUTION: Do not uncomment the below, instead change the component's + # "menu_label" setting. E.g.: + #pgwui_upload: + # menu_label: 'upload -- Upload File Into Database' + + # The order of the menu items can be manually specified based + # PGWUI component name. Omitted components come last. + #pgwui_menu: + # order: + # - 'pgwui_upload' + # - 'pgwui_logout' + + # pgwui_upload + + # The default pgwui_upload settings are: + #pgwui_upload: + # literal_column_headings: 'no-never' + # menu_label: 'upload -- Upload File Into Database' + # trim: 'choice-yes' + # null: 'choice-yes' + # file_format: 'csv' + # + # literal_column_headings + # Take uploaded column headings literally? + # The available choices are: + # yes-always The file's column headings, as typed, are the table's + # column names. + # choice-yes Present a checkbox, default to "yes". + # choice-no Present a checkbox, default to "no". + # no-never The file's column headings are given to PostgreSQL as-is, + # and so PostgreSQL normalizes them to lower case. + # Optional setting. The default is "no-never". + # + # Caution: Non-ASCII column names, particularly in the Turkish locale, + # are not guaranteed to be case-insensitive. + # + # trim + # Whether or not to remove leading and trailing whitespace from each + # uploaded data element. (From each "cell", each column of each row.) + # + # The available choices are: + # yes-always Always trim. + # choice-yes Present a checkbox, default to "yes, trim". (default) + # choice-no Present a checkbox, default to "no, don't trim". + # no-never Never trim. + # + # null + # Whether or not some uploaded data elements should be inserted as NULL + # values. + # + # The available choices are: + # yes-always Always transform some data into NULL. + # choice-yes Present a checkbox, default to "yes". + # choice-no Present a checkbox, default to "no". + # no-never Never transform data into NULL. + # + # file_format + # The format of the uploaded file + # + # The available choices are: + # csv The CSV format. + # tab Tab separated values. + + + # pgwui_bulk_upload + + # pgwui_bulk_upload: + # literal_column_headings: 'no-never' + # menu_label: 'bulk_upload -- Upload Many Files Into PostgreSQL' + # map_file: 'contents.yml' + # trim: 'choice-yes' + # null: 'choice-yes' + # file_format: 'csv' + # + # literal_column_headings: 'no-never' + # Take uploaded column headings literally? + # See pgwui_upload above for details. + # + # map_file + # Name of the file in the zip file that maps file names to table + # names. The default name of the map file is `contents.yml`. + # + # The map file is in YAML syntax, which for purposes of this document + # can be thought of as a superset of JSON. It must contain a top-level + # tag, `map_list`, which itself contains a list of maps, each of which + # maps a file to a table or view. The table (or view) names may, + # optionally, be schema qualified. An example map file might be: + # + # # This file is contents.yml + # map_list: + # # Load the foo.csv file into the foo_table table of the + # # default schema. + # - file_map: + # file: foo.csv + # relation: foo_table + # # Load the bar.csv file into the bar_view uploadable-view + # # of the default schema. + # - file_map: + # file: bar.csv + # relation: bar_view + # # Load the baz.csv file into the baz_table table of the meta schema. + # - file_map: + # file: baz.csv + # relation: meta.baz_table + # trim: false + # + # The files within each directory are uploaded in the order in which + # they are listed in the map file. The directories in the zip file are + # processed in alphabetical order. + # + # The "trim:" key in the file_map map controls whether or not whitespace + # is removed from data values. "trim:" takes a boolean value. It + # defaults to true. It overrides other choices in the GUI interface. + # + # Top level tags, other than the `map_list` tag, are ignored. + # + # It is recommended to enclose file names which contain spaces, or begin + # with a digit, in single quotes. But any YAML syntax is valid. + # + # trim: 'choice-yes' + # Remove leading and trailing whitespace. + # See pgwui_upload above for details. + # + # null: 'choice-yes' + # Insert NULL values. + # See pgwui_upload above for details. + # + # file_format: 'csv' + # The format of the uploaded files. + # See pgwui_upload above for details. + + + # pgwui_copy + + # pgwui_copy: + # menu_label: 'copy -- Copy a Schema Between Databases' + # + # default_source_db: (See description) + # The default for the database from which data is copied. + # The default is the value of "default_db" in the "pgwui" mapping. + # + # sensitive_dbs: (See description) + # A list of databases for which an extra + # confirmation step is required before alteration. Comparisons + # are done in a case-insensitive fashion. The default is the + # value of pgwui.default_db. The setting "" + # means that no database requires additional confirmation + # before alteration. Example: + # sensitive_dbs: + # - 'scratchdb' + # - 'scritchdb' + # + # default_target_db: '' + # The default for the database to which data is copied. (Optional) + # + # default_schema: '' + # The default for the name of the schema which is copied. (Optional) + # + # bin: '/usr/bin' + # Absolute path to the directory containing the pg_dump and pg_restore + # binaries. (Optional) + + + # + # Pyramid configuration + # + + # Pyramid's configuration does not use mappings. Instead it prefaces + # all of its setting names with "pyramid.". + pyramid.reload_templates: false + pyramid.debug_authorization: false + pyramid.debug_notfound: false + pyramid.debug_routematch: false + pyramid.default_locale_name: 'en' + pyramid.includes: [] + + # + # Beaker session management configuration + # https://beaker.readthedocs.io/en/latest/configuration.html + # + + # Beaker's configuration does not use mappings. Instead it prefaces + # all of its setting names with "session.". + session.type: 'memory' + session.lock_dir: '/var/lock/pgwui_server' + # Remove cookie in browser on browser close. (boolean) + session.cookie_expires: true + session.key: 'pgwui_server' + # HMAC secret + # (This should be set by you.) + #session.secret: 'xxxxxxrandomstring40characterslongxxxxxx' + # Send cookie only over https. (boolean) + # WARNING: To use HTTP, not HTTPS, "session.secure" must be false! + # This is the default, as the WSGI application is often + # expected to be run behind a reverse proxy, which does HTTPS to the client. + # CAUTION: If you are forcing the browser to use HTTPS you want + # "session.secure" to be true. + session.secure: false + # Sessions timeout after an hour if unused. + session.timeout: 3600 + # Pyramid sends cookies for exception pages (boolean) + session.cookie_on_exception: true + + + # + # Mako templates + # + # Fail when a variable reference is undefined. (The Mako default) (boolean) + #mako.strict_undefined: true + + +### +# wsgi server configuration +### + +server: + # Use waitress as the webserver, configuring the host and port, and + # configure a reverse-proxy to the network. + # + # An Nginx (https://www.nginx.org) reverse-proxy configuration might + # look like: + # + # location / { + # proxy_pass http://localhost:6543; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # } + # + use: egg:waitress#main + # (Because the "use:" invokes waitress, these are waitress configuration + # directives.) + # A value for "host" of 0.0.0.0 or "*", instead of "127.0.0.1", exposes the + # application to the network. Unless you trust everyone and every + # device that might access your network this will, at minimum, expose + # what could be highly sensitive information. See the documentation + # for more secure alternatives. + host: 127.0.0.1 + port: 6543 + + # When using a standalone WSGI server like uwsgi or Apache's mod_wsgi + # use the pgwui_server's WSGI. + #use: egg:PGWUI_Server#main + +### +# The DEFAULT section +# +# Paste-deploy passes the content of this mapping to all applications. +# +# Assuming the usual Pyramid coding idiom of def main(global_config, +# **settings): is used in your __init__.py. The global_config variable +# is then a dict containing the content of the config file's top-level +# DEFAULT key, if any. +### + +#DEFAULT: +# an_example_mapping: {example_key: example_value} + +### +# logging configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +# +# Configures logging via both syslog(3) and stdout, the latter being +# suitable for a systemd service. +# +# See also: +# https://docs.python.org/3/library/logging.config.html#logging-config-dictschema +### +logging: + version: 1 + disable_existing_loggers: false + root: + # Root logger + level: INFO + handlers: + - console + - syslog + loggers: + # These next mappings don't do anything as given. Because they have no + # handler they emit no messages. They are an example showing logging + # customized per pgwui component. You may want to add + # propagate: false + # keys to avoid duplicate logging if you customize them to + # emit log messages. + pgwui_server: + level: INFO + handlers: [] + qualname: pgwui_server + pgwui_upload: + level: INFO + handlers: [] + qualname: pgwui_upload + formatters: + console: + format: '%(asctime)s [%(levelname)s]: %(name)s - %(message)s' + handlers: + console: + class: logging.StreamHandler + level: INFO + stream: ext://sys.stdout + formatter: console + syslog: + class: logging.handlers.SysLogHandler + level: INFO + address: /dev/log + formatter: console + facility: LOG_USER diff --git a/examples/misc/development.yaml b/examples/misc/development.yaml new file mode 100644 index 0000000..25f2ef9 --- /dev/null +++ b/examples/misc/development.yaml @@ -0,0 +1,547 @@ +### +# PGWUI_Server configuration +# +# Configuration changes require a server restart! +# (But see the pserve command's --reload option.) + +# +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### + +app: + + # + # PasteDeploy configuration + # + + # https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/paste.html#pastedeploy-entry-points + use: 'egg:pgwui_server' + + # + # PGWUI configuration + # + + pgwui: + # Postgres connection configuration. + + # Both pgwui.pg_host and pgwui.pg_port are optional; + # they default to the Unix PG socket and the default Postgres port. + # An empty string for host means use the unix socket. + pg_host: '' + pg_port: '5432' + + # The database to use by default. + # Not having the setting is the same as "", the empty string. + #pgwui.default_db: '' + + # The (PostgreSQL-named) encoding used for strings on the client-side. + # The list of PostgreSQL encodings can be found at: + # https://www.postgresql.org/docs/current/multibyte.html#MULTIBYTE-CHARSET-SUPPORTED + # This setting defaults to "auto", the special PostgreSQL value which + # sets the client_encoding based on the encoding of the client + # system's locale. + # Set this value to "", the empty string, to use the server's encoding + # as the client encoding. (client_encoding: '') (or just omit the + # declaration) + #client_encoding: 'auto' + + + # How to link to the site's home page. Useful when the home page is + # not the PGWUI menu. + # + # This setting is a mapping (a dict). The keys depend on the "type" + # used. + # + # type: + # URL An URL. This is the default. + # file A file containing HTML, read from the file system. + # asset A Pyramid asset specification. Results in an URL + # which references a file which is part of a PGWUI + # component, or some other "static" file included in a Pyramid + # application. + # route A Pyramid route name. Results in an URL + # which references a page generated by a Pyramid application. + # "route_prefix" is not applied. + # + # + # When "type" is "URL", there are the following keys: + # + # source: (required) + # + # The default is ``/``, when there is no home_page setting. + # Which produces an URL with no "path". + # + # * A URI path beginning with ``/``. E.g.: '/home' + # + # * An URL without a protocol, so an URL beginning with ``//`` and + # followed by a domain. E.g.: //www.example.com The URL + # delivered to the browser contains the protocol used in the request. + # + # * An URL with a protocol. E.g.: https://www.example.com + # + # + # When "type" is "file", there are the following keys + # + # source: (required) + # A fully-qualified file system path, so a path beginning with + # a ``/``. E.g. /var/www/html/index.html + # Served with a content encoding of ``text/html``. + # + # url_path: (required) + # The "path" component of the URL used to retrieve the file. + # Must begin with a ``/``. "route_prefix" is not applied. + # + # When type is "asset", there are the following keys: + # + # source: (required) + # + # * A `Pyramid`_ `asset specification`_. It must reference a + # `static asset`_, a file included in a `Pyramid`_ application. + # A file containing a page of HTML. + # + # + # When type is "route", there are the following keys: + # + # source: (required) + # + # * A `Pyramid`_ `route name`_. Used to reach a page generated + # by a `Pyramid`_ application which uses `URL dispatch`_. + # + # + # This is the default: + #home_page: + # type: 'URL' + # source: '/' + + # How to link to a menu of PGWUI components. An alternative to using + # the PGWUI_Menu component. + # Configured as pgwui.home_page is configured, above. + # The default is to have no menu. + # + # The following example uses the URL without a path (e.g., + # http://www.example.com/) as the menu page: + # + # menu_page: + # type: 'URL' + # source: '/' + # + # The "menu_page" mapping overrides what is provided by the + # PGWUI_Menu component. + + # Whether to auto-discover the pgwui component modules. (optional) + # When false pgwui component names must be listed in + # pyramid.includes: ... + #autoconfigure: true + + # What PGWUI components and other pyramid modules to use. + # (Required when pgwui.autoconfigure is False, or you want the + # debug toolbar.) + # sample that includes the PGUI logout and upload components, + # in the block-style list format rather than the inline + # list format as appears in the Pyramid configuration section, below: + #pyramid.includes + # - 'pgwui_logout' + # - 'pgwui_upload' + + # Whether or not to change the db content. (required) + dry_run: false + + # routing + + # Routes are what call up specific pages. They are usually the + # part of the URL which comes after the http://example.com. + # + # All routes should probably begin with a "/" character but + # if they do not a "/" will be automatically prepended. + # + # A full URL may be given as a route. The is useful for redirecting + # to other sites. + # + # For more information on route syntax see: + # https://docs.pylonsproject.org/projects/pyramid/en/master/narr/urldispatch.html#route-pattern-syntax + + # A prefix for all routes. (optional) If the prefix is "/a/b/c" then + # all URLs will begin with something like: http://example.com/a/b/c + # The default is no prefix. + #route_prefix: '' + + # Overriding routes of specific PGWUI components + + # The syntax is name: "route", one per line. The "name" is the + # name of the PGWUI component. "route" is the route to use to access + # the component. So to access the logout page at + # http://example.com/logmeout the line would be: + # pgwui_logout: '/logmeout' + # + # Overriding routes is optional. + # + # The default for some PGWUI components are: + #routes: + # pgwui_copy: '/copy' + # pgwui_logout: '/logout' + # pgwui_upload: '/upload' + + # Overriding assets + # + # The visual presentation of a PGWUI component is controlled by its assets. + # Overriding assets allows for extensive customization of presentation. + # + # A Pyramid asset is any file included in a Pyramid application that is + # not a Python source code file. So assets are image files, Mako template + # files used to render pages, CSS files, JavaScript files, etc. + # + # See: + # https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/narr/assets.html + # + # By default no assets are overridden. Override assets with: + # + # override_assets: + # asset: 'new' + # + # 'asset' is the asset to override, a Pyramid asset specification + # + # 'new' is the new value either a Pyramid asset specification + # or a file system path + # + # Example, altering the menu presented by PGWUI_Menu: + # + # override_assets: + # 'pgwui_menu:templates/menu.mak': '/tmp/mymenu.mak' + + # Settings validation + + # Whether or not to validate the Beaker session.secret setting. (optional) + # session.secret must be valid to detect Cross-Site Request Forgery (CSRF) + # vulnerabilties. Validation is on by default. + # Turn validation off for debugging, no HMAC in use. + validate_hmac: false + + + # PGWUI Component Settings + + # Menu presentation + + # The PGWUI_Menu component automaticaly constructs a menu for the + # PGWUI components in use. The display value of the menu items can + # be overridden using a "menu_label" setting for each component. + # CAUTION: Do not uncomment the below, instead change the component's + # "menu_label" setting. E.g.: + #pgwui_upload: + # menu_label: 'upload -- Upload File Into Database' + + # The order of the menu items can be manually specified based + # PGWUI component name. Omitted components come last. + #pgwui_menu: + # order: + # - 'pgwui_upload' + # - 'pgwui_logout' + + # pgwui_upload + + # The default pgwui_upload settings are: + #pgwui_upload: + # literal_column_headings: 'no-never' + # menu_label: 'upload -- Upload File Into Database' + # trim: 'choice-yes' + # null: 'choice-yes' + # file_format: 'csv' + # + # literal_column_headings + # Take uploaded column headings literally? + # The available choices are: + # yes-always The file's column headings, as typed, are the table's + # column names. + # choice-yes Present a checkbox, default to "yes". + # choice-no Present a checkbox, default to "no". + # no-never The file's column headings are given to PostgreSQL as-is, + # and so PostgreSQL normalizes them to lower case. + # Optional setting. The default is "no-never". + # + # Caution: Non-ASCII column names, particularly in the Turkish locale, + # are not guaranteed to be case-insensitive. + # + # trim + # Whether or not to remove leading and trailing whitespace from each + # uploaded data element. (From each "cell", each column of each row.) + # + # The available choices are: + # yes-always Always trim. + # choice-yes Present a checkbox, default to "yes, trim". (default) + # choice-no Present a checkbox, default to "no, don't trim". + # no-never Never trim. + # + # null + # Whether or not some uploaded data elements should be inserted as NULL + # values. + # + # The available choices are: + # yes-always Always transform some data into NULL. + # choice-yes Present a checkbox, default to "yes". + # choice-no Present a checkbox, default to "no". + # no-never Never transform data into NULL. + # + # file_format + # The format of the uploaded file + # + # The available choices are: + # csv The CSV format. + # tab Tab separated values. + + + # pgwui_bulk_upload + + # pgwui_bulk_upload: + # literal_column_headings: 'no-never' + # menu_label: 'bulk_upload -- Upload Many Files Into PostgreSQL' + # map_file: 'contents.yml' + # trim: 'choice-yes' + # null: 'choice-yes' + # file_format: 'csv' + # + # literal_column_headings: 'no-never' + # Take uploaded column headings literally? + # See pgwui_upload above for details. + # + # map_file + # Name of the file in the zip file that maps file names to table + # names. The default name of the map file is `contents.yml`. + # + # The map file is in YAML syntax, which for purposes of this document + # can be thought of as a superset of JSON. It must contain a top-level + # tag, `map_list`, which itself contains a list of maps, each of which + # maps a file to a table or view. The table (or view) names may, + # optionally, be schema qualified. An example map file might be: + # + # # This file is contents.yml + # map_list: + # # Load the foo.csv file into the foo_table table of the + # # default schema. + # - file_map: + # file: foo.csv + # relation: foo_table + # # Load the bar.csv file into the bar_view uploadable-view + # # of the default schema. + # - file_map: + # file: bar.csv + # relation: bar_view + # # Load the baz.csv file into the baz_table table of the meta schema. + # - file_map: + # file: baz.csv + # relation: meta.baz_table + # trim: false + # + # The files within each directory are uploaded in the order in which + # they are listed in the map file. The directories in the zip file are + # processed in alphabetical order. + # + # The "trim:" key in the file_map map controls whether or not whitespace + # is removed from data values. "trim:" takes a boolean value. It + # defaults to true. It overrides other choices in the GUI interface. + # + # Top level tags, other than the `map_list` tag, are ignored. + # + # It is recommended to enclose file names which contain spaces, or begin + # with a digit, in single quotes. But any YAML syntax is valid. + # + # trim: 'choice-yes' + # Remove leading and trailing whitespace. + # See pgwui_upload above for details. + # + # null: 'choice-yes' + # Insert NULL values. + # See pgwui_upload above for details. + # + # file_format: 'csv' + # The format of the uploaded files. + # See pgwui_upload above for details. + + + # pgwui_copy + + # pgwui_copy: + # menu_label: 'copy -- Copy a Schema Between Databases' + # + # default_source_db: (See description) + # The default for the database from which data is copied. + # The default is the value of "default_db" in the "pgwui" mapping. + # + # sensitive_dbs: (See description) + # A list of databases for which an extra + # confirmation step is required before alteration. Comparisons + # are done in a case-insensitive fashion. The default is the + # value of pgwui.default_db. The setting "" + # means that no database requires additional confirmation + # before alteration. Example: + # sensitive_dbs: + # - 'scratchdb' + # - 'scritchdb' + # + # default_target_db: '' + # The default for the database to which data is copied. (Optional) + # + # default_schema: '' + # The default for the name of the schema which is copied. (Optional) + # + # bin: '/usr/bin' + # Absolute path to the directory containing the pg_dump and pg_restore + # binaries. (Optional) + + + # + # Pyramid configuration + # + + # Pyramid's configuration does not use mappings. Instead it prefaces + # all of its setting names with "pyramid.". + pyramid.reload_templates: true + pyramid.debug_authorization: true + pyramid.debug_notfound: true + pyramid.debug_routematch: true + pyramid.default_locale_name: 'en' + pyramid.includes: + - 'pyramid_debugtoolbar' + + # For turning off the toolbar + #pyramid.debug_authorization: false + #pyramid.debug_notfound: false + #pyramid.debug_routematch: false + + + # By default, the toolbar only appears for clients from IP addresses + # '127.0.0.1' and '::1'. + # debugtoolbar.hosts: + # - '127.0.0.1' + # - '::1' + + + # + # Beaker session management configuration + # https://beaker.readthedocs.io/en/latest/configuration.html + # + + # Beaker's configuration does not use mappings. Instead it prefaces + # all of its setting names with "session.". + session.type: 'memory' + session.lock_dir: '/var/lock/pgwui_server' + # Remove cookie in browser on browser close. + session.cookie_expires: true + session.key: 'pgwui_server' + # HMAC secret + # (This should be set by you.) + #session.secret: 'xxxxxxrandomstring40characterslongxxxxxx' + # Send cookie only over https + # WARNING: To use HTTP, not HTTPS, "session.secure" must be false! + # This is the default, as the WSGI application is often + # expected to be run behind a reverse proxy, which does HTTPS to the client. + # CAUTION: If you are forcing the browser to use HTTPS you want + # "session.secure" to be true. + session.secure: false + # Sessions timeout after an hour if unused. + session.timeout: 3600 + # Pyramid sends cookies for exception pages + session.cookie_on_exception: true + + + # + # Mako templates + # + # Fail when a variable reference is undefined. (The default) + #mako.strict_undefined: true + + +### +# wsgi server configuration +### + +server: + # Use waitress as the webserver, configuring the host and port, and + # configure a reverse-proxy to the network. + # + # An Nginx (https://www.nginx.org) reverse-proxy configuration might + # look like: + # + # location / { + # proxy_pass http://localhost:6543; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # } + # + use: egg:waitress#main + # (Because the "use:" invokes waitress, these are waitress configuration + # directives.) + # A value for "host" of 0.0.0.0 or "*", instead of "127.0.0.1", exposes the + # application to the network. Unless you trust everyone and every + # device that might access your network this will, at minimum, expose + # what could be highly sensitive information. See the documentation + # for more secure alternatives. + host: 127.0.0.1 + port: 6543 + + # When using a standalone WSGI server like uwsgi or Apache's mod_wsgi + # use the pgwui_server's WSGI. + #use: egg:PGWUI_Server#main + +### +# The DEFAULT section +# +# Paste-deploy passes the content of this mapping to all applications. +# +# Assuming the usual Pyramid coding idiom of def main(global_config, +# **settings): is used in your __init__.py. The global_config variable +# is then a dict containing the content of the config file's top-level +# DEFAULT key, if any. +### + +#DEFAULT: +# an_example_mapping: {example_key: example_value} + +### +# logging configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +# +# Emit all logs, and stdout from the application, to stderr. +# Configured so that logs are viewable from the terminal window used +# to run pserve. +# +# See also: +# https://docs.python.org/3/library/logging.config.html#logging-config-dictschema +### +logging: + version: 1 + disable_existing_loggers: false + root: + # Root logger + level: DEBUG + handlers: + - console + # - syslog + loggers: + # These next mappings don't do anything as given. Because they have no + # handler they emit no messages. They are an example showing logging + # customized per pgwui component. You may want to add + # propagate: false + # keys to avoid duplicate logging if you customize them to + # emit log messages. + pgwui_server: + level: DEBUG + handlers: [] + qualname: pgwui_server + pgwui_upload: + level: DEBUG + handlers: [] + qualname: pgwui_upload + formatters: + console: + format: '%(asctime)s [%(levelname)s]: %(name)s - %(message)s' + handlers: + console: + class: logging.StreamHandler + level: INFO + stream: ext://sys.stdout + formatter: console +# # syslog: +# # class: logging.handlers.SysLogHandler +# # level: INFO +# # address: /dev/log +# # formatter: console +# # facility: LOG_USER diff --git a/pyproject.toml b/pyproject.toml index 278304f..f313865 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,3 +89,7 @@ classifiers = [ # Its configuration is manually coded. [project.entry-points.'paste.app_factory'] 'main' = 'pgwui_server.pgwui_server:main' + +# Use yaml configuration +[project.entry-points.'plaster.loader_factory'] +'file+yaml' = 'plaster_yaml:Loader' diff --git a/setup.py b/setup.py index 5ac003c..e178767 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def filter_readme(): # install_requires = [ 'pgwui_common==' + version, + 'plaster-yaml', 'pyramid', ] diff --git a/src/pgwui_server/exceptions.py b/src/pgwui_server/exceptions.py index 19990df..445f251 100644 --- a/src/pgwui_server/exceptions.py +++ b/src/pgwui_server/exceptions.py @@ -38,19 +38,6 @@ class AutoconfigureConflict(ServerInfo): 'Autoconfigure is True and there is a pyramid.include setting') -class MissingEqualError(SetupError): - def __init__(self, line): - super().__init__( - 'Expecting text containing an equals (=) sign, but ' - f'instead got ({line})') - - -class BadValueError(SetupError): - def __init__(self, setting, ex): - super().__init__( - f'Bad setting value supplied to ({setting}): {ex}') - - class BadSettingsAbort(SetupError): def __init__(self): super().__init__('Aborting due to bad setting(s)') diff --git a/src/pgwui_server/pgwui_server.py b/src/pgwui_server/pgwui_server.py index 2efa4ac..680a3ce 100644 --- a/src/pgwui_server/pgwui_server.py +++ b/src/pgwui_server/pgwui_server.py @@ -39,7 +39,7 @@ import pgwui_common.urls # Constants -# All the single-valued settings recognized by PGWUI_Server/Core +# All the settings recognized by PGWUI_Server/Core SETTINGS = set( ['pg_host', 'pg_port', @@ -49,11 +49,7 @@ SETTINGS = set( 'route_prefix', 'validate_hmac', 'autoconfigure', - ]) - -# All the multi-valued settings recognized by PGWUI_Server/Core -MULTI_SETTINGS = set( - ['routes', + 'routes', 'home_page', 'menu_page', 'override_assets', @@ -63,8 +59,8 @@ MULTI_SETTINGS = set( DEFAULT_HOME_PAGE_TYPE = 'URL' DEFAULT_HOME_PAGE_SOURCE = '/' DEFAULT_SETTINGS = { # As delivered by configparser to this parser - 'pgwui.home_page': f'type = {DEFAULT_HOME_PAGE_TYPE}\n' - f'source = {DEFAULT_HOME_PAGE_SOURCE}\n'} + 'home_page': {'type': DEFAULT_HOME_PAGE_TYPE, + 'source': DEFAULT_HOME_PAGE_SOURCE}} # Logging log = logging.getLogger(__name__) @@ -72,104 +68,27 @@ log = logging.getLogger(__name__) # Functions -def dot_to_dict(settings, key, new_key): - settings['pgwui'][new_key] = settings[key] - del settings[key] - - -def parse_multiline_assignments(lines, result): - '''Add the parse value to the result - ''' - for line in lines.splitlines(): - if '=' in line: - key, val = line.split('=', 1) - result.append((key.rstrip(), val.lstrip())) - else: - stripped = line.lstrip() - if stripped != '': - # Multiple values on different lines means a list - try: - key, val = result[-1] - except IndexError: - raise server_ex.MissingEqualError(stripped) - if not isinstance(val, list): - val = [val] - val.append(stripped) - result[-1] = (key, val) - - -def parse_assignments(lines): - '''Return a list of key/value tuples from the lines of a setting - ''' - result = [] - if isinstance(lines, str): - parse_multiline_assignments(lines, result) - else: - for key, val in lines.items(): - result.append((key, val)) - return result - - -def dot_to_multiline_setting(settings, key, pgwui_key): - '''Put a multi-line setting into its own dict, - adding to what's already there +def check_component_settings( + errors, component_checkers, pgwui_settings, component): + '''Validate the component's settings ''' - multi_setting = settings['pgwui'].setdefault(pgwui_key, dict()) - try: - multi_setting.update(dict(parse_assignments(settings[key]))) - except server_ex.MissingEqualError: - raise - finally: - del settings[key] - - -def component_setting_into_dict( - errors, component_checkers, key, settings, component): - '''Put a component's settings in its own dict and validate them - ''' - try: - dot_to_multiline_setting(settings, key, component) - except server_ex.MissingEqualError as ex: - # Couldn't get the settings because there's no "=" - errors.append(server_ex.BadValueError(f'pgwui:{component}', ex)) - return if component in component_checkers: errors.extend( - component_checkers[component](settings['pgwui'][component])) + component_checkers[component](pgwui_settings[component])) -def setting_into_dict( - errors, components, component_checkers, key, settings): - '''Separate a pgwui setting into a dict on '.' chars; validate - component settings. - ''' - if key[:6] == 'pgwui.': - new_key = key[6:] - if new_key in components: - component_setting_into_dict( - errors, component_checkers, key, settings, new_key) - else: - if new_key in SETTINGS: - dot_to_dict(settings, key, new_key) - elif new_key in MULTI_SETTINGS: - try: - dot_to_multiline_setting(settings, key, new_key) - except server_ex.MissingEqualError as ex: - errors.append( - server_ex.BadValueError(f'pgwui:{new_key}', ex)) - else: - errors.append(common_ex.UnknownSettingKeyError(key)) - - -def dictify_settings(errors, settings, components): - '''Convert "." in the pgwui settings to dict mappings, and validate - the result. +def check_all_components_settings(errors, settings, components): + '''Validate the settings of the PGWUI compoenents. ''' component_checkers = plugin.find_pgwui_check_settings() - settings.setdefault('pgwui', dict()) - for key in list(settings.keys()): - setting_into_dict( - errors, components, component_checkers, key, settings) + + pgwui_settings = settings['pgwui'] + for component in list(pgwui_settings): + if component in components: + check_component_settings( + errors, component_checkers, pgwui_settings, component) + elif component not in SETTINGS: + errors.append(common_ex.UnknownSettingKeyError(component)) def exit_reporting_errors(errors): @@ -190,8 +109,10 @@ def exit_reporting_errors(errors): def add_default_settings(settings): '''Add the default settings to the config if not there ''' + settings.setdefault('pgwui', dict()) + pgwui_settings = settings['pgwui'] for setting, val in DEFAULT_SETTINGS.items(): - settings.setdefault(setting, val) + pgwui_settings.setdefault(setting, val) def exit_on_invalid_settings(settings, components): @@ -199,8 +120,8 @@ def exit_on_invalid_settings(settings, components): ''' add_default_settings(settings) errors = [] - dictify_settings(errors, settings, components) - check_settings.validate_settings(errors, settings) + check_all_components_settings(errors, settings, components) + check_settings.validate_settings(errors, settings) # check common settings if errors: exit_reporting_errors(errors) diff --git a/tests/test_pgwui_server.py b/tests/test_pgwui_server.py index 50e6312..cd8edb1 100644 --- a/tests/test_pgwui_server.py +++ b/tests/test_pgwui_server.py @@ -36,7 +36,6 @@ import pgwui_common.urls from pgwui_develop import testing import pgwui_server.pgwui_server as pgwui_server -import pgwui_server.exceptions as server_ex # Activiate the PGWUI pytest plugin pytest_plugins = ("pgwui",) @@ -68,76 +67,20 @@ mock_set_urls = testing.make_mock_fixture( # Unit tests -# dot_to_multiline_setting() +# check_component_settings() -def test_dot_to_multiline_setting_new(mock_parse_assignments): - '''Adds a new dict and puts the settings in it - ''' - comp_settings = {'foo': 'foo', 'bar': 'bar'} - component = 'pgwui_component' - key = 'pgwui.' + component - settings = {'pgwui': {}, - key: comp_settings} - expected = {'pgwui': {component: comp_settings}} - - mock_parse_assignments.return_value = comp_settings - pgwui_server.dot_to_multiline_setting(settings, key, component) - - assert settings == expected - - -def test_dot_to_multiline_setting_old(mock_parse_assignments): - '''Extends an existing dict in the settings - ''' - comp_settings = {'foo': 'foo', 'bar': 'bar'} - component = 'pgwui_component' - key = 'pgwui.' + component - settings = {'pgwui': {component: {'foo': 'bar', 'baz': 'baz'}}, - key: comp_settings} - expected = {'pgwui': - {component: {'foo': 'foo', 'bar': 'bar', 'baz': 'baz'}}} - - mock_parse_assignments.return_value = comp_settings - pgwui_server.dot_to_multiline_setting(settings, key, component) - - assert settings == expected - - -def test_dot_to_multiline_setting_bad(mock_parse_assignments): - '''When the value is bad we get the expected error - ''' - component = 'pgwui_component' - key = 'pgwui.' + component - settings = {'pgwui': {}, - key: 'ignored'} - - mock_parse_assignments.side_effect = server_ex.MissingEqualError('text') - with pytest.raises(server_ex.MissingEqualError): - pgwui_server.dot_to_multiline_setting(settings, key, component) - - assert True - - -mock_dot_to_multiline_setting = testing.make_mock_fixture( - pgwui_server, 'dot_to_multiline_setting') - - -# component_setting_into_dict() - -def test_component_setting_into_dict_no_checker( - mock_dot_to_multiline_setting): +def test_check_component_settings_no_checker(): '''When there's no checker nothing is done ''' errors = [] - pgwui_server.component_setting_into_dict( - errors, {}, 'pgwui.pgwui_component', None, 'pgwui_component') + pgwui_server.check_component_settings( + errors, {}, None, 'pgwui_component') assert errors == [] -def test_component_setting_into_dict_checker( - mock_dot_to_multiline_setting): +def test_check_component_settings_checker(): '''When there's a checker its result is appended to the errors ''' errors = ['someerror'] @@ -145,242 +88,62 @@ def test_component_setting_into_dict_checker( expected = copy.deepcopy(errors) expected.extend(new_errors) - pgwui_server.component_setting_into_dict( + pgwui_server.check_component_settings( errors, {'pgwui_component': lambda settings: new_errors}, - 'pgwui.pgwui_component', {'pgwui': {'pgwui_component': {}}}, - 'pgwui_component') + {'pgwui_component': {}}, 'pgwui_component') assert errors == expected -def test_component_setting_into_dict_nosettings( - mock_dot_to_multiline_setting): - '''When there's no settings due to a syntax error the right error - is appended to the errors - ''' - errors = [] - mock_dot_to_multiline_setting.side_effect = server_ex.MissingEqualError(0) - - pgwui_server.component_setting_into_dict( - errors, {}, 'pgwui.pgwui_component', None, 'pgwui_component') - - assert len(errors) == 1 - assert isinstance(errors[0], server_ex.BadValueError) - - -mock_component_setting_into_dict = testing.make_mock_fixture( - pgwui_server, 'component_setting_into_dict') - - -# dot_to_dict() - -def test_dot_to_dict(): - '''Removes pgwui.* settings, replaces them with a dict entry - ''' - settings = {'foo': 1, - 'pgwui': {}, - 'pgwui.abc': 'abc', - 'pgwui.def': 'def'} - expected = {'foo': 1, - 'pgwui': {'abc': 'abc'}, - 'pgwui.def': 'def'} - - pgwui_server.dot_to_dict(settings, 'pgwui.abc', 'abc') - - assert settings == expected - - -mock_dot_to_dict = testing.make_mock_fixture( - pgwui_server, 'dot_to_dict') - - -# parse_multiline_assigments() - - -def test_parse_multiline_assignments_str(): - '''Appends key/value string tuples and when there's no "=", - and more than just whitespace, a list is the result - ''' - lines = ('key1 = value1\n' # whitespace around = is ignored - '\n' - 'second\n' - 'third\n' - 'key2=value2\n' # missing whitespace is fine - 'key3= value3=withequals\n' - ) - result = [] - pgwui_server.parse_multiline_assignments(lines, result) - assert result == [('key1', ['value1', 'second', 'third']), - ('key2', 'value2'), - ('key3', 'value3=withequals')] - - -def test_parse_multiline_assignments_no_equal(): - '''When the line contains no equal sign the right exception is raised - ''' - with pytest.raises(server_ex.MissingEqualError): - pgwui_server.parse_multiline_assignments('noequal\n', []) - - -mock_parse_multiline_assignments = testing.make_mock_fixture( - pgwui_server, 'parse_multiline_assignments') - - -# parse_assignments() - -def test_parse_assignments_str(mock_parse_multiline_assignments): - '''Calls parse_multiline_assignments''' - lines = ('key1 = value1\n' # whitespace around = is ignored - '\n' - 'ignored\n' - 'key2=value2\n' # missing whitespace is fine - 'key3= value3=withequals\n' - ) - pgwui_server.parse_assignments(lines) - mock_parse_multiline_assignments.assert_called_once() - - -def test_parse_assignments_dict(mock_parse_multiline_assignments): - '''Returns key value tuples. - ''' - lines = {'key1': 'value1', - 'key2': 'value2', - } - result = pgwui_server.parse_assignments(lines) - assert set(result) == set([('key1', 'value1'), - ('key2', 'value2'), - ]) - - -mock_parse_assignments = testing.make_mock_fixture( - pgwui_server, 'parse_assignments') +mock_check_component_settings = testing.make_mock_fixture( + pgwui_server, 'check_component_settings') -# setting_into_dict() +# check_all_components_settings() -def test_setting_into_dict_unknown( - mock_component_setting_into_dict, - mock_dot_to_dict, - mock_dot_to_multiline_setting): - '''No new errors when there's a non-pgwui setting''' - errors = [] - pgwui_server.setting_into_dict(errors, [], {}, 'foo', {}) - - assert errors == [] - - -def test_setting_into_dict_bad( - mock_parse_assignments, - mock_component_setting_into_dict, - mock_dot_to_dict, - mock_dot_to_multiline_setting): +def test_check_all_components_settings_bad( + mock_check_component_settings): '''Delivers an error on a bad pgwui setting''' errors = [] - pgwui_server.setting_into_dict( - errors, [], {}, 'pgwui.foo', {}) + pgwui_server.check_all_components_settings( + errors, {'pgwui': {'foo': {}}}, {}) assert errors assert isinstance(errors[0], common_ex.UnknownSettingKeyError) -def test_setting_into_dict_good( - mock_component_setting_into_dict, - mock_dot_to_dict, - mock_dot_to_multiline_setting): - '''Calls dot_to_dict when a known pgwui setting is supplied - ''' - errors = [] - - pgwui_server.setting_into_dict( - errors, [], {}, 'pgwui.pg_host', {}) - - mock_dot_to_dict.assert_called_once() - mock_dot_to_multiline_setting.assert_not_called() - assert errors == [] - - -def test_setting_into_dict_multiline( - mock_component_setting_into_dict, - mock_dot_to_dict, - mock_dot_to_multiline_setting): - '''Calls dot_to_multiline_setting when a known pgwui multi-line - setting is supplied +def test_check_all_components_settings_good( + mock_check_component_settings): + '''Produces no error when a known core pgwui setting is supplied ''' errors = [] - pgwui_server.setting_into_dict( - errors, [], {}, 'pgwui.home_page', {}) - - mock_dot_to_dict.assert_not_called() - mock_dot_to_multiline_setting.assert_called_once() - assert errors == [] - + pgwui_server.check_all_components_settings( + errors, {'pgwui': {'pg_host': 'example.com'}}, {}) -def test_setting_into_dict_plugin_component( - mock_component_setting_into_dict, - mock_dot_to_dict, - mock_dot_to_multiline_setting): - '''When a setting is for a component the setting is parsed and - moved into a dict - ''' - key = 'pgwui.pgwui_component' - settings = {key: None} - errors = [] - mock_parse_assignments.return_value = {} + mock_check_component_settings.assert_not_called() - pgwui_server.setting_into_dict( - errors, ['pgwui_component'], {}, key, settings) - - mock_component_setting_into_dict.assert_called_once() assert errors == [] -def test_setting_into_dict_bad_assignment( - mock_parse_assignments, - mock_component_setting_into_dict, - mock_dot_to_dict, - mock_dot_to_multiline_setting): - '''Delivers an error on a setting that has no "=" +def test_check_all_components_settings_checked( + mock_check_component_settings): + '''Produces no error when a known pgwui_component setting is supplied, + and checks the component's settings ''' errors = [] - mock_dot_to_multiline_setting.side_effect = server_ex.MissingEqualError(0) - - pgwui_server.setting_into_dict( - errors, [], {}, 'pgwui.home_page', {}) - mock_component_setting_into_dict.assert_not_called() - mock_dot_to_dict.assert_not_called() - mock_dot_to_multiline_setting.assert_called_once() + pgwui_server.check_all_components_settings( + errors, {'pgwui': {'pgwui_menu': {}}}, {'pgwui_menu': {}}) - assert len(errors) == 1 - assert isinstance(errors[0], server_ex.BadValueError) + mock_check_component_settings.assert_called_once() - -mock_setting_into_dict = testing.make_mock_fixture( - pgwui_server, 'setting_into_dict') - - -# dictify_settings() - -def test_dictify_settings(mock_find_pgwui_check_settings, - mock_setting_into_dict): - '''Calls setting_into_dict() for each key in setting, - with the proper list of plugin components - ''' - settings = {'key1': 'value1', - 'key2': 'value2'} - components = ['pgwui_server'] - - errors = [] - pgwui_server.dictify_settings(errors, settings, components) - - assert mock_setting_into_dict.call_count == len(settings) - assert mock_setting_into_dict.call_args[0][1] == components + assert errors == [] -mock_dictify_settings = testing.make_mock_fixture( - pgwui_server, 'dictify_settings') +mock_check_all_components_settings = testing.make_mock_fixture( + pgwui_server, 'check_all_components_settings') # exit_reporting_errors() @@ -451,7 +214,7 @@ def test_add_default_settings(): settings = dict() pgwui_server.add_default_settings(settings) - assert settings == pgwui_server.DEFAULT_SETTINGS + assert settings == {'pgwui': pgwui_server.DEFAULT_SETTINGS} mock_add_default_settings = testing.make_mock_fixture( @@ -462,15 +225,15 @@ mock_add_default_settings = testing.make_mock_fixture( def test_exit_on_invalid_settings_invalid( monkeypatch, - mock_add_default_settings, mock_dictify_settings, + mock_add_default_settings, mock_check_all_components_settings, mock_validate_settings, mock_exit_reporting_errors): - '''Calls dictify_settings(), validate_settings(), and then + '''Calls check_all_components_settings(), validate_settings(), and then exit_reporting_errors() when setting is invalid ''' def mymock(errors, settings, components): errors.append('error1') - mock_dictify_settings.side_effect = mymock + mock_check_all_components_settings.side_effect = mymock mock_exit_reporting_errors.side_effect = lambda *args: sys.exit(1) with pytest.raises(SystemExit) as excinfo: @@ -478,14 +241,14 @@ def test_exit_on_invalid_settings_invalid( assert excinfo[1].code == 1 - mock_dictify_settings.assert_called_once() + mock_check_all_components_settings.assert_called_once() mock_validate_settings.assert_called_once() mock_add_default_settings.assert_called_once() assert mock_exit_reporting_errors.called def test_exit_on_invalid_settings_valid( - mock_add_default_settings, mock_dictify_settings, + mock_add_default_settings, mock_check_all_components_settings, mock_validate_settings, mock_exit_reporting_errors): '''Returns, without exiting, when all settings are valid ''' diff --git a/tests/test_pgwui_server_integration.py b/tests/test_pgwui_server_integration.py index 721c6a0..66b20d1 100644 --- a/tests/test_pgwui_server_integration.py +++ b/tests/test_pgwui_server_integration.py @@ -27,6 +27,7 @@ import pgwui_menu import pgwui_server.pgwui_server as pgwui_server +import copy # Mark all tests as "integrationtest" pytestmark = pytest.mark.integrationtest @@ -34,33 +35,37 @@ pytestmark = pytest.mark.integrationtest # Constants TEST_SETTINGS = { - 'pgwui.validate_hmac': 'False', - 'pgwui.dry_run': 'False', + 'pgwui': { + 'validate_hmac': 'False', + 'dry_run': 'False', + } } REFERENCE_SETTINGS = { - 'pgwui.pg_host': '', - 'pgwui.pg_port': '5432', - 'pgwui.default_db': 'template1', - 'pgwui.autoconfigure': 'True', - 'pgwui.dry_run': 'False', - 'pgwui.validate_hmac': 'False', - # 'pgwui.home_page': { # The default - # 'type': 'URL', - # 'source': '/'}, - # 'pgwui.menu_page': { # An example - # 'type': 'URL', - # 'source': '/'}, - # 'pgwui.route_prefix': '', # The default - # 'pgwui.routes': { # An example - # 'pgwui_logout': '/logout', - # 'pgwui_upload': '/upload'} - # 'pgwui.pgwui_menu': { # An example - # 'order': ['pgwui_upload', - # 'pgwui_logout'], - # 'pgwui_upload': { # The defaults - # 'literal_column_headings': 'off', - # 'menu_label': 'upload -- Upload File Into Database'}, + 'pgwui': { + 'pg_host': '', + 'pg_port': '5432', + 'default_db': 'template1', + 'autoconfigure': 'True', + 'dry_run': 'False', + 'validate_hmac': 'False', + # 'home_page': { # The default + # 'type': 'URL', + # 'source': '/'}, + # 'menu_page': { # An example + # 'type': 'URL', + # 'source': '/'}, + # 'route_prefix': '', # The default + # 'routes': { # An example + # 'pgwui_logout': '/logout', + # 'pgwui_upload': '/upload'} + # 'pgwui_menu': { # An example + # 'order': ['pgwui_upload', + # 'pgwui_logout'], + # 'pgwui_upload': { # The defaults + # 'literal_column_headings': 'off', + # 'menu_label': 'upload -- Upload File Into Database'}, + } } @@ -81,8 +86,8 @@ def check_route(config, name, expected): def updated_dict(old, new): - updated = old.copy() - updated.update(new) + updated = copy.deepcopy(old) + updated['pgwui'].update(new) return updated @@ -93,14 +98,14 @@ def updated_dict(old, new): pgwui_menu.pgwui_menu.DEFAULT_MENU_ROUTE, None), (updated_dict(REFERENCE_SETTINGS, - {'pgwui.route_prefix': '/foo', - 'pgwui.menu_page': {'type': 'file', - 'source': '/tmp/nofile.html', - 'url_path': '/menu'}}), + {'route_prefix': '/foo', + 'menu_page': {'type': 'file', + 'source': '/tmp/nofile.html', + 'url_path': '/menu'}}), 'foo' + pgwui_logout.pgwui_logout.DEFAULT_LOGOUT_ROUTE, 'foo' + pgwui_menu.pgwui_menu.DEFAULT_MENU_ROUTE, '/menu')]) -def test_pgwui_server_config_no_route_prefix( +def test_pgwui_server_config_route_prefix( settings, logout_path, menu_path, menu_page_path): '''The given route_prefix is applied to the routes ''' diff --git a/tox.ini b/tox.ini index 96f4815..ffed379 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ deps = check-manifest cmarkgfm flake8 + yamllint pytest pytest-cov twine @@ -26,6 +27,7 @@ commands = python setup.py sdist twine check dist/* flake8 . + yamllint --strict . py.test -m unittest --cov=pgwui_server tests/ py.test -m 'not unittest' --cov=pgwui_server tests/ # coverage run --source src/pgwui_server -m py.test -- 2.34.1