From 0f634a634d88c9043f0ac07920f20e563808b97c Mon Sep 17 00:00:00 2001 From: "Karl O. Pinc" Date: Mon, 18 Sep 2023 16:37:29 -0500 Subject: [PATCH] Support including view definitions into the documentation --- .gitignore | 3 ++ db/README | 21 +++++++++++++ doc/README | 8 +++++ doc/extract_view.py | 65 +++++++++++++++++++++++++++++++++++++++++ make_files/make_docs.mk | 44 ++++++++++++++++++++++++++-- 5 files changed, 138 insertions(+), 3 deletions(-) create mode 100755 doc/extract_view.py diff --git a/.gitignore b/.gitignore index 92d79c4..9d8a7f9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ db/pg_settings.new # Built image files doc/images/ +# Built sql which becomes views +doc/views/ + # Built documentation doc/sphinx-doc/ diff --git a/db/README b/db/README index ea4946e..6419663 100644 --- a/db/README +++ b/db/README @@ -26,6 +26,9 @@ qualify the created object with a schema. The schema into which each object is placed is determined by the placement of the file which creates it in the build system's directory structure. +Some statements must be written in a particular fashion or installing, +or re-installing objects into the database will fail. These are: + CREATE FUNCTION to be written CREATE OR REPLACE FUNCTION.... DROP FUNCTION to be written DROP FUNCTION IF EXISTS.... @@ -39,6 +42,24 @@ DROP TYPE to be written DROP TYPE IF EXISTS.... CREATE SCHEMA to be written CREATE SCHEMA IF NOT EXISTS.... +Further, some statements must match particular patterns, and be on +their own line. Indentation is allowed. If this is done improperly +views will not appear appropriately in the documentation. The +statements, and their form, are: + +CREATE VIEW something AS +-- Body of view goes here, on a new line, for as many lines as necessary, +-- and ends in a semicolon. Comments are allowed. + +-- CREATE VIEW can be followed by GRANT, and should not be immediately +-- followed by any other statement. The GRANT statement must look +-- like: + +GRANT -- With a trailing space after the word. +-- Then the rest of the grant statement, for as many lines as needed. +-- (Usually grants are all on one line and generated by macros.) + + Notes: UNIQUE constraints are not used in CREATE TABLE statements. Instead diff --git a/doc/README b/doc/README index cc63b6c..029948c 100644 --- a/doc/README +++ b/doc/README @@ -4,6 +4,14 @@ extensions. There are a number of conventions: The section "underline" hierarchy is as given in the RST docs: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#sections +To include the statement used to generate a particular view use: + +.. include:: /views/$SCHEMA/$VIEWNAME.sql + :code: sql + +Where $SCHEMA is the schema the view is in and $VIEWNAME is the name +of the view. + Each table, and each column, has replacement text defined in epilog.rst.m4. This is so that the section headings can have "extra comment" in them. (Simplest to just always have replacement text, rather than worrying about diff --git a/doc/extract_view.py b/doc/extract_view.py new file mode 100755 index 0000000..98cee2a --- /dev/null +++ b/doc/extract_view.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +# Copyright (C) 2023 The Meme Factory, Inc. http://www.karlpinc.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +'''Read from stdin and (heuristically) extract the SQL that makes up +the body of the view. +''' +# Karl O. Pinc +# +import re + +# Patterns +CREATE_VIEW = re.compile(r"\s*CREATE OR REPLACE VIEW \S+ AS\s*$") +GRANT = re.compile(r"\s*GRANT ") +SQL_COMMENT = re.compile(r"\s*(?:--.*)|$") + +def get_body(fd): + '''Return a list of every line between (exclusive) the line matching + CREATE_VIEW and the line matching GRANT. + ''' + body = [] + found = False + for line in fd: + if found: + if GRANT.match(line) is not None: + break + body.append(line) + else: + found = CREATE_VIEW.match(line) is not None + return body + +def cleanup(body): + '''Mutate the list that is body, with leading and trailing lines that + are empty or consist only of whitespace removed, and all trailing + lines that match SQL_COMMENT removed. + ''' + while body[0].strip == '': + del body[0] + + for pos in range(len(body) - 1, -1, -1): + if SQL_COMMENT.match(body[pos]) is None: + break + del body[pos] + +def main(): + with open('/dev/stdin', 'r') as fd: + body = get_body(fd) + cleanup(body) + for line in body: + print(line, end="") + +if __name__ == "__main__": + main() diff --git a/make_files/make_docs.mk b/make_files/make_docs.mk index 46fabd7..80c57e4 100644 --- a/make_files/make_docs.mk +++ b/make_files/make_docs.mk @@ -92,6 +92,9 @@ HTML_SRCDIR := $(SPHINX_DOCDIR)/$(HTML_SRCDIR) HTML_IMAGEDIR := $(HTML_SRCDIR)/images LATEX_SRCDIR := $(SPHINX_DOCDIR)/latex_src LATEX_IMAGEDIR := $(LATEX_SRCDIR)/images +VIEW_DIR := $(DOCDIR)/views +HTML_VIEW_DIR := $(HTML_SRCDIR)/views +LATEX_VIEW_DIR := $(LATEX_SRCDIR)/views # Locations of m4 macros M4_DOC_INCLUDE_PATH := doc/include @@ -101,9 +104,20 @@ M4_DOC_INCLUDE_ARGS := -I $(M4_DOC_INCLUDE_PATH) -I $(M4_GLOBAL_INCLUDE_PATH) PDF_A4_TARGET := $(DOCDIR)/$(PDF_A4_NAME) PDF_LETTER_TARGET := $(DOCDIR)/$(PDF_LETTER_NAME) +# The source SQL files that create the views. +VIEW_SOURCE := $(patsubst %.m4,%.sql, \ + $(wildcard db/schemas/*/views/create/*.m4)) +# The SQL files that contain the body of the views. +VIEW_SQL := $(subst /views/create,, \ + $(subst db/schemas,$(VIEW_DIR),$(VIEW_SOURCE))) + # Rebuild when any of these change +# (Depend on all of the views, on the assumption that they'll all +# be documented. If not, do a little extra work. Too bad. +# Likewise, rebuild all the docs if any of the views change.) DOC_DEPENDS := $(GENERIC_DEPENDS) \ $(wildcard $(M4_DOC_INCLUDE_PATH)/*.m4) \ + $(VIEW_SQL) \ $(DOCDIR)/conf.py # Computed variables @@ -175,10 +189,11 @@ CLEAN_DOC_TARGETS := $(DOCDIR)/requirements.txt.new \ # Prerequsites for html generation HTML_PREREQS := install-sphinx html_rst $(HTML_IMAGE_FILES) $(HTML_EPILOG) \ - $(HTML_STATIC_DIR) $(HTML_IMAGEDIR) + $(HTML_STATIC_DIR) $(HTML_IMAGEDIR) $(HTML_VIEW_DIR) # Prerequsites for latex (pdf) generation -LATEX_PREREQS := latex_rst $(PNG_FILES) $(LATEX_EPILOG) $(LATEX_IMAGEDIR) +LATEX_PREREQS := latex_rst $(PNG_FILES) $(LATEX_EPILOG) $(LATEX_IMAGEDIR) \ + $(LATEX_VIEW_DIR) ## ## The available targets for make (make TARGET) are: @@ -344,7 +359,7 @@ $(PDF_LETTER_PATH): $(PDF_LETTER_TARGET) # Construct all the source dirs used to build output $(LATEX_DIRS) $(HTML_DIRS) $(HTML_SRCDIR) $(LATEX_SRCDIR) \ - $(HTML_STATIC_DIR) $(DOC_ROOT) $(IMAGES_BASEDIR): + $(HTML_STATIC_DIR) $(DOC_ROOT) $(IMAGES_BASEDIR) $(VIEW_DIR): mkdir -p $@ # Sphinx requires all images be located in the $(SPHINX_DOCDIR), @@ -354,6 +369,10 @@ $(LATEX_DIRS) $(HTML_DIRS) $(HTML_SRCDIR) $(LATEX_SRCDIR) \ $(HTML_IMAGEDIR) $(LATEX_IMAGEDIR): ln -s ../../images $@ +# Likewise with the SQL of view bodies +$(HTML_VIEW_DIR) $(LATEX_VIEW_DIR): + ln -s ../../views $@ + # # The files sphinx builds for a pdf # It is unclear whether it's a good idea to run latexmk ourselves @@ -389,6 +408,16 @@ $(LATEX_EPILOG): $(SOURCE_EPILOG) $(DOC_DEPENDS) $< \ > $@ +# Targets that make the views from their m4 files. +# In theory, this would be useful for all database objects, but +# we only _need_ the views for the docs so that's all we do. +$(VIEW_SOURCE): + $(MAKE) TARGET_SCHEMA=$$(basename \ + $$(dirname $$(dirname $$(dirname $@)))) \ + TARGET_TYPE=views/create \ + TARGET_THING=$$(basename -s .sql $@) \ + sql_file + # # Allow automatic variables to be used in prerequsites # @@ -424,3 +453,12 @@ $(LATEX_RST_FILES): $$(patsubst %.rst,%.m4,\ $(patsubst %.rst,%.m4,\ $(subst $(LATEX_SRCDIR),$(STOCK_SRCDIR),$@)) \ > $@ + +# The "body" of the CREATE VIEW statements. +$(VIEW_SQL): db/schemas/$$(notdir $$(patsubst \ + %/,%,$$(dir $$@)))/views/create/$$(notdir $$@) \ + $(VIEW_DIR) doc/extract_view.py + mkdir -p $(VIEW_DIR)/$$(basename $$(dirname $@)) + doc/extract_view.py \ + < $< \ + > $@ -- 2.34.1