From 0c4ddd897a6b5b482f8b8d8e2fca0ac91f4ae33f Mon Sep 17 00:00:00 2001 From: "Karl O. Pinc" Date: Tue, 9 May 2023 11:49:33 -0500 Subject: [PATCH] Build system for text docs -- images not yet supported --- .gitignore | 7 + Makefile | 336 +++++++++++++++++++++++++++++++++++ db/include/constants.m4 | 34 ++++ db/include/macros.m4 | 45 +++++ doc/conf.py | 81 +++++++++ doc/include/constants.m4 | 42 +++++ doc/include/macros.m4 | 80 +++++++++ doc/src/epilog.inc.m4 | 46 +++++ doc/src/er_diagrams.m4 | 51 ++++++ doc/src/er_diagrams/key.m4 | 32 ++++ doc/src/index.m4 | 44 +++++ doc/src/intro.m4 | 33 ++++ doc/src/intro/about_db.m4 | 50 ++++++ doc/src/intro/about_doc.m4 | 59 ++++++ doc/src/intro/permissions.m4 | 59 ++++++ help.mk | 21 +++ include/constants.m4 | 37 ++++ include/macros.m4 | 40 +++++ localize.mk | 23 +++ rwildcard.mk | 28 +++ 20 files changed, 1148 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 db/include/constants.m4 create mode 100644 db/include/macros.m4 create mode 100644 doc/conf.py create mode 100644 doc/include/constants.m4 create mode 100644 doc/include/macros.m4 create mode 100644 doc/src/epilog.inc.m4 create mode 100644 doc/src/er_diagrams.m4 create mode 100644 doc/src/er_diagrams/key.m4 create mode 100644 doc/src/index.m4 create mode 100644 doc/src/intro.m4 create mode 100644 doc/src/intro/about_db.m4 create mode 100644 doc/src/intro/about_doc.m4 create mode 100644 doc/src/intro/permissions.m4 create mode 100644 help.mk create mode 100644 include/constants.m4 create mode 100644 include/macros.m4 create mode 100644 localize.mk create mode 100644 rwildcard.mk diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..232d3aa --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# Built documentation +doc/html_epilog.inc +doc/latex_epilog.inc +doc/sphinx-doc/ + +# The python venv used by sphinx +doc/sphinx-venv/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f4bdd94 --- /dev/null +++ b/Makefile @@ -0,0 +1,336 @@ +# Copyright (C) 2019, 2023 The Meme Factory, Inc. www.karlpinc.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +# To get a database of comments, look for: +# ^.. |TABLEORTABLECOLUMN| replace:: .* +# Save in a directory structure, each file named with TABLEORTABLECOLUMN +# and the content is ".*". +# What do we do to get rid of markup in the replacement text? +# Run it through `sphinx-build -b text ...` (ignoring link errors) +# Parse it ourselves +# Then query the db for all tables and columns (and functions), delete +# all the comments for each and insert new ones. + +# Make "help" be the default target +include help.mk + +# Functions +include rwildcard.mk +include localize.mk + +# Conditionals +HTML_LOCAL_SRC := html_local_src +HTML_SRC := html_src + +## +## Variables: +## +## LOCALHTML +## To generate html output and access it via "file://" URLs define this to +## something, e.g.:: make LOCALHTML=y html +ifdef LOCALHTML +ifndef SDB_HTML_FMT +SDB_HTML_FMT := localhtml +endif +ifndef HTML_SRCDIR +HTML_SRCDIR := $(HTML_LOCAL_SRC) +endif +BUILDERNAME := html +else +ifndef SDB_HTML_FMT +SDB_HTML_FMT := html +endif +ifndef HTML_SRCDIR +HTML_SRCDIR := $(HTML_SRC) +endif +BUILDERNAME := dirhtml +endif + + +# Variables +PDF_NAME := sokwedb_technical.pdf + +DOC_ROOT := /var/www/html +PDF_PATH := $(DOC_ROOT)/$(PDF_NAME) +HTML_PATH := $(DOC_ROOT)/doc + +PREFIX := doc +SPHINX_PREFIX := $(PREFIX)/sphinx-doc +SPHINX_BUILDDIR := $(SPHINX_PREFIX)/build +SPHINX_BUILT_LATEX := $(SPHINX_BUILDDIR)/latex +SPHINX_BUILT_HTML := $(SPHINX_BUILDDIR)/html + +STOCK_SRCDIR := $(PREFIX)/src +HTML_STATIC_DIR := $(PREFIX)/_static +HTML_SRCDIR := $(SPHINX_PREFIX)/$(HTML_SRCDIR) +LATEX_SRCDIR := $(SPHINX_PREFIX)/latex_src + +M4_INCLUDE_PATH := include +M4_DOC_INCLUDE_PATH := doc/include +M4_DB_INCLUDE_PATH := db/include + +EMACS_BK_CHR := ~ + +# The target sphinx builds for a pdf +PDF_TARGET := $(PDF_NAME) + +# Rebuild when any of these change +DB_DEPENDS := Makefile $(wildcard $(M4_INCLUDE_PATH)/*.m4) \ + $(wildcard $(M4_DB_INCLUDE_PATH)/*.m4) +DOC_DEPENDS := Makefile $(wildcard $(M4_INCLUDE_PATH)/*.m4) \ + $(wildcard $(M4_DOC_INCLUDE_PATH)/*.m4) \ + $(PREFIX)/conf.py + +# Computed variables +SPHINX_VENV := $(PREFIX)/sphinx-venv +SPHINXBUILD := $(SPHINX_VENV)/bin/sphinx-build -n -c $(PREFIX) + +M4_BASEDIR := $(STOCK_SRCDIR) + +# Intermediate variables for filtering and transforming paths + +M4_FILES := $(call localize,$(call rwildcard,$(M4_BASEDIR)/,*.m4)) +M4_FILES := $(filter-out %epilog.inc.m4, $(M4_FILES)) + +SOURCE_FILES = $(call localize,$(call rwildcard,$(M4_BASEDIR)/,*)) + +SOURCE_BASE = $(filter-out %$(EMACS_BK_CHR) $(M4_FILES), $(SOURCE_FILES)) + +HTML_FILES := $(call localize,$(call rwildcard,$(HTML_SRCDIR)/,*)) + +LATEX_FILES := $(call localize,$(call rwildcard,$(LATEX_SRCDIR)/,*)) + +# The html files to build and the directories that contain the files +HTML_RST_FILES := \ + $(patsubst %.m4,%.rst,\ + $(subst /$(STOCK_SRCDIR)/,/$(HTML_SRCDIR)/,$(M4_FILES))) +HTML_DIRS := $(sort $(dir $(HTML_RST_FILES))) + +# The latex files to build and the directories that contain the files +LATEX_RST_FILES := \ + $(patsubst %.m4,%.rst,\ + $(subst /$(STOCK_SRCDIR)/,/$(LATEX_SRCDIR)/,$(M4_FILES))) +LATEX_DIRS := $(sort $(dir $(LATEX_RST_FILES))) + +# The epilog +SOURCE_EPILOG := $(M4_BASEDIR)/epilog.inc.m4 +HTML_EPILOG := $(PREFIX)/html_epilog.inc +LATEX_EPILOG := $(PREFIX)/latex_epilog.inc + +# Where sphinx puts the pdf it builds +SPHINX_PDF_TARGET := $(SPHINX_BUILT_LATEX)/$(PDF_TARGET) + +# Image targets +DIA_BASEDIR := diagrams/dia/ +IMAGES_BASEDIR := images/ +DIA_FILES := $(wildcard $(DIA_BASEDIR)*.dia) + +PARTIAL_PATHS := $(subst $(DIA_BASEDIR),$(IMAGES_BASEDIR), $(DIA_FILES)) +SVG_FILES := $(patsubst %.dia,%.svg,$(PARTIAL_PATHS)) +PNG_FILES := $(patsubst %.dia,%.png,$(PARTIAL_PATHS)) + +ifdef LOCALHTML +HTML_IMAGE_FILES := $(PNG_FILES) +else +HTML_IMAGE_FILES := $(SVG_FILES) +endif + +CLEAN_TARGETS := $(SPHINX_PREFIX) \ + $(HTML_RST_FILES) $(LATEX_RST_FILES) \ + $(HTML_EPILOG) $(LATEX_EPILOG) \ + $(SVG_FILES) $(PNG_FILES) \ + $(PDF_TARGET) + +## +## The available targets for make (make TARGET) are: +## + +## install Build all the docs and install on the website +.PHONY: install +install: install-html install-pdf + +## html Make the html docs with sphinx +.PHONY: html +html: install-sphinx html_rst $(HTML_IMAGE_FILES) $(HTML_EPILOG) \ + $(HTML_STATIC_DIR) + $(SPHINXBUILD) -t html_epilog -b $(BUILDERNAME) \ + $(HTML_SRCDIR) $(SPHINX_BUILT_HTML) + +## pdf Make pdf docs with sphinx +.PHONY: pdf +pdf: install-sphinx $(PDF_TARGET) + +## mostlyclean Delete all the generated files but the virtualenv +.PHONY: mostlyclean +mostlyclean: + rm -rf $(CLEAN_TARGETS) + +## clean Delete all user generated files +.PHONY: clean +clean: mostlyclean destroy-sphinx-venv + + +## +## Lesser used targets are: +## + +## install-pdf Install the pdf docs on the website +.PHONY: install-pdf +install-pdf: $(PDF_PATH) + +## install-html Install the html docs on the website +.PHONY: install-html +install-html: html + rsync -rtO --delete --force $(SPHINX_BUILT_HTML)/ $(HTML_PATH) + +## install-sphinx Install sphinx in a virtualenv +.PHONY: install-sphinx +install-sphinx: sphinx-venv $(SPHINX_VENV)/bin/sphinx-build + +## report-old Report on outdated packages in the sphinx venv +.PHONY: report-old +report-old: + $(SPHINX_VENV)/bin/pip list -o + +## update-sphinx Update sphinx venv content to the latest versions +.PHONY: update-sphinx +update-sphinx: destroy-sphinx-venv install-sphinx + +## sphinx-venv Make a virtual environment for sphinx to run from +.PHONY: sphinx-venv +sphinx-venv: $(SPHINX_VENV) + +$(SPHINX_VENV): + python3 -m venv $(SPHINX_VENV) + python3 -m venv --upgrade-deps $(SPHINX_VENV) + +## destroy-sphinx-venv Delete the sphinx virtual environment +.PHONY: destroy-sphinx-venv +destroy-sphinx-venv: + rm -rf $(SPHINX_VENV) + +## html_rst Build RST files from m4 source for HTML generation +.PHONY: html_rst +html_rst: $(HTML_DIRS) $(HTML_RST_FILES) $(HTML_SRCDIR) + # Get rid of anything not a *.rst file, not already in the source + rm -rf \ + $(filter-out \ + $(HTML_RST_FILES) \ + $(subst /$(STOCK_SRCDIR)/,/$(HTML_SRCDIR)/,$(SOURCE_BASE)) \ + $(HTML_DIRS) \ + , $(HTML_FILES)) + # Populate a sourcedir for HTML output, without the RST files + rsync -a --exclude='*.m4' --exclude='*$(EMACS_BK_CHR)' \ + $(STOCK_SRCDIR)/ \ + $(HTML_SRCDIR) + +## latex_rst Build RST files from m4 source for LaTeX generation +.PHONY: latex_rst +latex_rst: $(LATEX_DIRS) $(LATEX_RST_FILES) $(LATEX_SRCDIR) + # Get rid of anything not a *.rst file, not already in the source + rm -rf $(filter-out \ + $(LATEX_RST_FILES) \ + $(subst \ + /$(STOCK_SRCDIR)/,/$(LATEX_SRCDIR)/,$(SOURCE_BASE)) \ + $(LATEX_DIRS) \ + , $(LATEX_FILES)) + # Populate a sourcedir for LATEX output, without the RST files + rsync -a --exclude='*.m4' --exclude='*$(EMACS_BK_CHR)' \ + $(STOCK_SRCDIR)/ \ + $(LATEX_SRCDIR) + +## png_files Build *.png files from dia source +.PHONY: png_files +png_files: $(PNG_FILES) + +## svg_files Build *.svg files from dia source +.PHONY: svg_files +svg_files: $(SVG_FILES) + +# +# Internal targets +# + +# Ensure that sphinx is installed +$(SPHINX_VENV)/bin/sphinx-build: + $(SPHINX_VENV)/bin/pip install sphinx + +# Install the pdf on the website +$(PDF_PATH): $(PDF_TARGET) + rsync -a $(PDF_TARGET) $(PDF_PATH) + +# Construct all the source dirs used to build output +$(LATEX_DIRS) $(HTML_DIRS) $(HTML_SRCDIR) $(LATEX_SRCDIR) \ + $(HTML_STATIC_DIR): + mkdir -p $@ + +# The file we want for a pdf +$(PDF_TARGET): $(SPHINX_PDF_TARGET) + cp $(PREFIX)/build/latex/$(PDF_TARGET) $(PDF_TARGET) + +# The file sphinx builds for a pdf +$(SPHINX_PDF_TARGET): latex_rst $(PNG_FILES) $(LATEX_EPILOG) + $(SPHINXBUILD) -t latex_epilog -b latexpdf \ + $(LATEX_SRCDIR) $(SPHINX_BUILT_LATEX) + +# The rst_epilogs of conf.py. + +$(HTML_EPILOG): $(SOURCE_EPILOG) $(DOC_DEPENDS) + m4 -I $(M4_DOC_INCLUDE_PATH) \ + --define=sdb_output_fmt=$(SDB_HTML_FMT) \ + $< \ + > $@ +$(LATEX_EPILOG): $(SOURCE_EPILOG) $(DOC_DEPENDS) + m4 -I $(M4_DOC_INCLUDE_PATH) \ + --define=sdb_output_fmt=latex \ + $< \ + > $@ + +# +# Allow automatic variables to be used in prerequsites +# +.SECONDEXPANSION: + +# Construct *.png files +$(SVG_FILES): $$(patsubst %.svg,%.dia,\ + $$(subst $(IMAGES_BASEDIR),$(DIA_BASEDIR),$$@)) \ + $(DOC_DEPENDS) + dia --export $@ $< + +# Construct *.png files +$(PNG_FILES): $$(patsubst %.png,%.dia,\ + $$(subst $(IMAGES_BASEDIR),$(DIA_BASEDIR),$$@)) \ + $(DOC_DEPENDS) + dia --export $@ $< + +# Build all the RST for html output +$(HTML_RST_FILES): $$(patsubst %.rst,%.m4,\ + $$(subst $(HTML_SRCDIR),$(STOCK_SRCDIR),$$@)) \ + $(DOC_DEPENDS) + m4 -I $(M4_DOC_INCLUDE_PATH) \ + --define=sdb_output_fmt=$(SDB_HTML_FMT) \ + $< \ + > $@ + +# Build all the RST for latex output +$(LATEX_RST_FILES): $$(patsubst %.rst,%.m4,\ + $$(subst $(LATEX_SRCDIR),$(STOCK_SRCDIR),$$@)) \ + $(DOC_DEPENDS) + m4 -I $(M4_DOC_INCLUDE_PATH) \ + --define=gdb_output_fmt=latex \ + $(patsubst %.rst,%.m4,\ + $(subst $(LATEX_SRCDIR),$(STOCK_SRCDIR),$@)) \ + > $@ diff --git a/db/include/constants.m4 b/db/include/constants.m4 new file mode 100644 index 0000000..8cf5428 --- /dev/null +++ b/db/include/constants.m4 @@ -0,0 +1,34 @@ +dnl Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU Affero General Public License as +dnl published by the Free Software Foundation, either version 3 of the +dnl License, or (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU Affero General Public License for more details. +dnl +dnl You should have received a copy of the GNU Affero General Public License +dnl +dnl Establish symbols for constants used througout the database. +dnl +dnl +changequote([{[,]}])dnl m4 foolery so includes include only once. +dnl Once the macro is in the text, change the quotes back +ifdef([{[_db_constants.m4]}], [{[changequote(`,')]}], [{[dnl (see bottom) +changequote(`,')dnl +dnl +dnl Standard test for having already included the file. +define(`_db_constants.m4')dnl +dnl +dnl Have m4 discard output generated in this file (see bottom) +divert(`-1') + +dnl +dnl General symbols +dnl + +divert(`0')dnl Output with m4 again +]}])dnl End of ifdef over the whole file. diff --git a/db/include/macros.m4 b/db/include/macros.m4 new file mode 100644 index 0000000..38d6c1b --- /dev/null +++ b/db/include/macros.m4 @@ -0,0 +1,45 @@ +dnl Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU Affero General Public License as +dnl published by the Free Software Foundation, either version 3 of the +dnl License, or (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU Affero General Public License for more details. +dnl +dnl You should have received a copy of the GNU Affero General Public License +dnl +dnl Macros used throughout the system. +dnl +dnl This should be included first so that the divert macro requires a +dnl parameter. +dnl +dnl For safety, prefix all macros with "jgi_". +dnl +dnl +changequote([{[,]}])dnl m4 foolery so includes include only once. +dnl Once the macro is in the text, change the quotes back +ifdef([{[_db_macros.m4]}], [{[changequote(`,')]}], [{[dnl (see bottom) +changequote(`,')dnl +dnl +dnl Standard test for having already included the file. +define(`_db_macros.m4')dnl +dnl +dnl Have m4 discard output generated in this file (see bottom) +divert(`-1') + +dnl Include standard macros +include(include/macros.m4) + +define(`jgi_generated_sql', +`-- DO NOT EDIT THIS FILE. It was automatically generated. Edit +-- the *.m4 files instead. (Files _should_ be re-created by +-- typing "make" at the command line.) +') + + +divert(`0')dnl Output with m4 again +]}])dnl End of ifdef over the whole file. diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..1a811bd --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,81 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'SokweDB Technical Documentation' +copyright = '2019, 2023 The Meme Factory, Inc.' +author = 'Karl O. Pinc, The Meme Factory, Inc. kop@karlpinc.com' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# Epilog to add to all RST files. Puts a footer on and defines substitutions. +# +if tags.has('html_epilog'): + epilog_path = 'html_epilog.inc' +if tags.has('latex_epilog'): + epilog_path = 'latex_epilog.inc' +with open(epilog_path, 'r') as epilog: + rst_epilog = epilog.read() + +# -- Options for HTML output ------------------------------------------------- + +# Do not put '.html' on the end of links +#html_link_suffix = '' + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Customize theme +html_theme_options = { + 'extra_nav_links': {'HOME': '/'}, + 'show_related': True, + 'show_relbars': True, + 'sidebar_collapse': False, + } + +# Do not provide RST source +html_copy_source = False + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%Y-%m-%d %H:%M:%S%z' +# %z does not work in sphinx-build 2.2.1 +today_fmt = '%Y-%m-%d %H:%M:%S' + diff --git a/doc/include/constants.m4 b/doc/include/constants.m4 new file mode 100644 index 0000000..9cba59b --- /dev/null +++ b/doc/include/constants.m4 @@ -0,0 +1,42 @@ +dnl Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU Affero General Public License as +dnl published by the Free Software Foundation, either version 3 of the +dnl License, or (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU Affero General Public License for more details. +dnl +dnl You should have received a copy of the GNU Affero General Public License +dnl +dnl Establish symbols for constants used througout the database. +dnl +dnl +changequote([{[,]}])dnl m4 foolery so includes include only once. +dnl Once the macro is in the text, change the quotes back +ifdef([{[_doc_constants.m4]}], [{[changequote(`,')]}], [{[dnl (see bottom) +changequote(`,')dnl +dnl +dnl Standard test for having already included the file. +define(`_doc_constants.m4')dnl +dnl +dnl Have m4 discard output generated in this file (see bottom) +divert(`-1') + +dnl Include shared constants +include(include/constants.m4) + +dnl +dnl Symbols for transforming paths to images, which vary by the +dnl type of documentation generated. +dnl + +dnl The extension (png, svg, etc.) to use when referencing image files. +define(`sdb_html_image_extension', `svg') +define(`sdb_latex_image_extension', `png') + +divert(`0')dnl Output with m4 again +]}])dnl End of ifdef over the whole file. diff --git a/doc/include/macros.m4 b/doc/include/macros.m4 new file mode 100644 index 0000000..82766fb --- /dev/null +++ b/doc/include/macros.m4 @@ -0,0 +1,80 @@ +dnl Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU Affero General Public License as +dnl published by the Free Software Foundation, either version 3 of the +dnl License, or (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU Affero General Public License for more details. +dnl +dnl You should have received a copy of the GNU Affero General Public License +dnl +dnl Macros used throughout the system. +dnl +dnl This should be included first so that the divert macro requires a +dnl parameter. +dnl +dnl For safety, prefix all macros with "sdb_". +dnl +dnl +changequote([{[,]}])dnl m4 foolery so includes include only once. +dnl Once the macro is in the text, change the quotes back +ifdef([{[_doc_macros.m4]}], [{[changequote(`,')]}], [{[dnl (see bottom) +changequote(`,')dnl +dnl +dnl Standard test for having already included the file. +define(`_doc_macros.m4')dnl +dnl +dnl Have m4 discard output generated in this file (see bottom) +divert(`-1') + +dnl Include standard macros +dnl include(include/macros.m4) + +dnl Define special quotes for RST documents because they use the +dnl backtick character in their syntax. +define(`sdb_rst_quotes', + `ifelse(`$1', `on', + `changequote(`*m4[',`]m4*')', + `changequote()')') + +dnl Issue a "generated" warning in the generated document. +define(`sdb_generated_rst', +`.. DO NOT EDIT THIS FILE. It was automatically generated. Edit + the *.m4 files instead. (Files _should_ be re-created by + typing "make" at the command line.) +') + +dnl +dnl Transformations depending on output format +dnl + +dnl Conditional for testing html output +define(`sdb_if_htmlout', + `ifelse(sdb_output_fmt, latex, $2, $1)') + +dnl Use a macro to transform image paths since this is "easier". + +dnl Transform a path to an image so that it is suitable for html output +define(`sdb_html_er_image_path', `../../$1.sdb_html_image_extension') + +dnl Transform a path to an image so that it is suitable for latex output +define(`sdb_latex_er_image_path', `../../$1.sdb_latex_image_extension') + +dnl sdb_output_fmt +dnl This is dynamically defined when building to control output +ifelse(sdb_output_fmt, `latex', + `define(`sdb_er_image_path', `sdb_latex_er_image_path')', + sdb_output_fmt, `localhtml', + `define(`sdb_er_image_path', `sdb_latex_er_image_path')', + `define(`sdb_er_image_path', `sdb_html_er_image_path')') + +dnl sdb_image_path +dnl This is dynamically defined when building to be one of the 2 macros above. + + +divert(`0')dnl Output with m4 again +]}])dnl End of ifdef over the whole file. diff --git a/doc/src/epilog.inc.m4 b/doc/src/epilog.inc.m4 new file mode 100644 index 0000000..eca9966 --- /dev/null +++ b/doc/src/epilog.inc.m4 @@ -0,0 +1,46 @@ +.. Copyright (C) 2019, 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4)dnl +include(macros.m4)dnl +sdb_rst_quotes(`on')dnl +sdb_generated_rst()dnl + +.. +.. Elements included on every page +.. + +.. URLs of external pages. + +.. _UUID: https://en.wikipedia.org/wiki/Universally_unique_identifier + +.. _technical debt: https://en.wiktionary.org/wiki/technical_debt + +.. _Entity Relationship Diagrams: + https://en.wikipedia.org/wiki/Entity%E2%80%93relationship_model + + +.. Substitutions (used to give labels to internal links) + +.. For lookup tables -- the "value" column + .. |Valuecol| replace:: :ref:`Value ` + +.. |allownull| replace:: This column may be ``NULL``. + +.. The upload schema, which is not yet documented + .. |Uploads| replace:: :ref:`Uploads ` + +sdb_if_htmlout(*m4[Page generated on: |today|]m4*, *m4[]m4*) diff --git a/doc/src/er_diagrams.m4 b/doc/src/er_diagrams.m4 new file mode 100644 index 0000000..93e6502 --- /dev/null +++ b/doc/src/er_diagrams.m4 @@ -0,0 +1,51 @@ +.. Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4) +include(macros.m4) +sdb_rst_quotes(`on') +sdb_generated_rst() + + +.. _erdiagrams: + +Entity Relationship Diagrams +============================ + +`Entity Relationship Diagrams`_, or ER Diagrams, are graphic pictures +of how rows in the database's tables relate to other rows. They are +dense in information about what data exists and in what tables it can +be found. + +For example, individuals can transfer between communities zero or more +times. A single individual is one kind of entity recorded in the +database, one transfer of a single individual between communities is a +second kind of entity. Each individual is represented in the database +as a single row in a table. Each transfer is likewise represented by +a single row in a different table. + +The ER diagram of this shows that a row representing a one individual +can be "connected to" zero or more rows recording community +transfers. The diagram also shows that a transfer **must** be +"connected to" exactly one individual, the individual who is changing +communities. A transfer should not be able to exist unless the +individual transferring also exists. + + +.. toctree:: + :maxdepth: 3 + + er_diagrams/key.rst diff --git a/doc/src/er_diagrams/key.m4 b/doc/src/er_diagrams/key.m4 new file mode 100644 index 0000000..dd269e0 --- /dev/null +++ b/doc/src/er_diagrams/key.m4 @@ -0,0 +1,32 @@ +.. Copyright (C) 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4) +include(macros.m4) +sdb_rst_quotes(`on') +sdb_generated_rst() + + +Key +--- + +.. Commented out figure + .. figure:: sdb_er_image_path()(images/diagram_key) + :alt: Key to symbols used on the entity relationship diagrams + + Key to the Entity Relationship Diagrams + +The system is not yet displaying images in the docs. diff --git a/doc/src/index.m4 b/doc/src/index.m4 new file mode 100644 index 0000000..b6f020b --- /dev/null +++ b/doc/src/index.m4 @@ -0,0 +1,44 @@ +.. Copyright (C) 2019, 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4)dnl +include(macros.m4)dnl +sdb_rst_quotes(`on')dnl +sdb_generated_rst()dnl + +####### +SokweDB +####### +####################### +Technical Documentation +####################### + +.. toctree:: + :maxdepth: sdb_if_htmlout(3, 5) + + intro.rst + er_diagrams.rst + +.. Add these back later.... + schemas.rst + appendices.rst + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/doc/src/intro.m4 b/doc/src/intro.m4 new file mode 100644 index 0000000..3a40190 --- /dev/null +++ b/doc/src/intro.m4 @@ -0,0 +1,33 @@ +.. Copyright (C) 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4) +include(macros.m4) +sdb_rst_quotes(`on') +sdb_generated_rst() + +Introduction +============ + +This documentation describes the design and capabilities of the +SokweDB database and related systems. + +.. toctree:: + :maxdepth: 3 + + intro/about_doc.rst + intro/about_db.rst + intro/permissions.rst diff --git a/doc/src/intro/about_db.m4 b/doc/src/intro/about_db.m4 new file mode 100644 index 0000000..f25475f --- /dev/null +++ b/doc/src/intro/about_db.m4 @@ -0,0 +1,50 @@ +.. Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4) +include(macros.m4) +sdb_rst_quotes(`on') +sdb_generated_rst() + + +About the Database +------------------ + +Databases store data in tables. Related singleton datum, such as a +single name, a single birthdate, a mother, are kept together in a +single row of a table constructed to hold this particular kind of +data. Data of the same kind kept within a single table are are found +in a column of the table. Columns have names, like "name", +"birthdate", and "mother_id". + +So a table is a grid containing (classically) a single value in each +cell of the grid. Each row of the table represents a physical thing, +such as a chimpanzee, or an abstract thing, such as the distance to some +designated chimpanzee. E.g., a row with the 2 columns: a distance in +meters, and the id of the 2nd chimpanzee. Each column of a table is +expected to contain the same "kind" of data; a name should not go in +the "birthdate"column. + +.. _views: + +Views appear to be tables but they are not. Views are virtual, when +queried they deliver the results of a query run against the database's +actual tables. When setup to do so, changing the data in a view can +change data in the database's underlying tables. + +The coordinate reference system, e.g. the geodetic datum ``WGS 1984``, +of the data stored in the database's latitude and longitude +measurements is *not* known. diff --git a/doc/src/intro/about_doc.m4 b/doc/src/intro/about_doc.m4 new file mode 100644 index 0000000..f2d3fe1 --- /dev/null +++ b/doc/src/intro/about_doc.m4 @@ -0,0 +1,59 @@ +.. Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4) +include(macros.m4) +sdb_rst_quotes(`on') +sdb_generated_rst() + +About This Document +------------------- + +The database is case insensitive when it comes to the names used to +identify content -- table names and column names and so forth.\ +[#f6]_ Within this document schema names are written with an +initial capital letter, table and view names are written in all upper +case, and column names are written with initial capital letters. + +Unless otherwise noted all columns must contain a value. Note that an +empty string, the series of characters consisting of zero characters, +is a value. The empty string\ [#f4]_ is a string and so it is +reasonable to compare it with other non-empty strings. The empty +string is distinct from ``NULL``\ [#f5]_, which means "no +information", and is used when there is no value/no data at all. + +Data consistency is guaranteed only when the guarantee is explicitly +mentioned.\ [#f15]_ Care must be taken when updating data unless there +is an explicit guarantee of data consistently. + +This document uses a particular vocabulary, which informs the table +names within the database. The selection of the vocabulary is based on +the terminology in use at the JGI and may need some study for those +not familiar with it. + + +.. rubric:: Footnotes + +.. [#f6] To be precise, *unless special steps are taken* the database, + and SQL, is case-insensitive when it comes to names. + +.. [#f4] Written ``''`` in SQL. + +.. [#f5] Written ``NULL`` in SQL. + +.. [#f15] The notations in the :ref:`erdiagrams` are dense in + constraints, constraints built into the database, which + ensure data integrity. diff --git a/doc/src/intro/permissions.m4 b/doc/src/intro/permissions.m4 new file mode 100644 index 0000000..09dc3d3 --- /dev/null +++ b/doc/src/intro/permissions.m4 @@ -0,0 +1,59 @@ +.. Copyright (C) 2019, 2023 The Meme Factory, Inc. www.karlpinc.com + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +.. M4 setup +include(constants.m4) +include(macros.m4) +sdb_rst_quotes(`on') +sdb_generated_rst() + + +Database Permissions +-------------------- + +To access the data in the database permission must be +granted. This is done per user login. + +There are 2 ordinary levels of permission. Their names are: + +sdb_reader + Permission to query database content. + +sdb_writer + All the permissions of ``sdb_reader`` plus permission to alter + the content of the database. + +Ordinary permissions are database :dfn:`roles`. They can be granted +with SQL, e.g.:: + + GRANT sdb_reader TO someuser; + +Or grants can be made through some other mechanism. + +Superusers +^^^^^^^^^^ + +`Superusers`_ have permission to do anything with a database, create +and destroy tables, create and destroy user logins, etc. Only a few +people are expected to have superuser privileges. + +Those people with superuser privileges will typically have 2 logins, +one ordinary login and a second login with superuser privileges. The +superuser login should be used only when necessary, as when a new +person is given access to SokweDB and a new database login must be +created. Ordinary interactions with the database, data entry, data +retrieval, etc., should be done with a non-superuser login. + +.. _Superusers: https://www.postgresql.org/docs/current/role-attributes.html diff --git a/help.mk b/help.mk new file mode 100644 index 0000000..9893c10 --- /dev/null +++ b/help.mk @@ -0,0 +1,21 @@ +# Generate help summary from Makefile content +# Copyright (C) 2019 The Meme Factory, Inc. www.karlpinc.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Use this by putting the following line into your Makefile: +# include help.mk + +help: + @grep -Eh '^##($$| )' $(MAKEFILE_LIST) | sed -E 's/^##($$| )//' diff --git a/include/constants.m4 b/include/constants.m4 new file mode 100644 index 0000000..720fcd8 --- /dev/null +++ b/include/constants.m4 @@ -0,0 +1,37 @@ +dnl Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU Affero General Public License as +dnl published by the Free Software Foundation, either version 3 of the +dnl License, or (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU Affero General Public License for more details. +dnl +dnl You should have received a copy of the GNU Affero General Public License +dnl +dnl Establish symbols for constants used througout the database. +dnl +dnl +changequote([{[,]}])dnl m4 foolery so includes include only once. +dnl Once the macro is in the text, change the quotes back +ifdef([{[_constants.m4]}], [{[changequote(`,')]}], [{[dnl (see bottom) +changequote(`,')dnl +dnl +dnl Standard test for having already included the file. +define(`_constants.m4')dnl +dnl +dnl Have m4 discard output generated in this file (see bottom) +divert(`-1') + +dnl +dnl Database permissions +dnl + +define(`sdb_reader', `reader') +define(`sdb_writer', `writer') + +divert(`0')dnl Output with m4 again +]}])dnl End of ifdef over the whole file. diff --git a/include/macros.m4 b/include/macros.m4 new file mode 100644 index 0000000..2a032c0 --- /dev/null +++ b/include/macros.m4 @@ -0,0 +1,40 @@ +dnl Copyright (C) 2019, 2020, 2023 The Meme Factory, Inc. www.karlpinc.com +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU Affero General Public License as +dnl published by the Free Software Foundation, either version 3 of the +dnl License, or (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU Affero General Public License for more details. +dnl +dnl You should have received a copy of the GNU Affero General Public License +dnl +dnl Macros used throughout the system. +dnl +dnl This should be included first so that the divert macro requires a +dnl parameter. +dnl +dnl For safety, prefix all macros with "sdb_". +dnl +dnl +changequote([{[,]}])dnl m4 foolery so includes include only once. +dnl Once the macro is in the text, change the quotes back +ifdef([{[_macros.m4]}], [{[changequote(`,')]}], [{[dnl (see bottom) +changequote(`,')dnl +dnl +dnl Standard test for having already included the file. +define(`_macros.m4')dnl +dnl +dnl Have m4 discard output generated in this file (see bottom) +divert(`-1') + +dnl Require that the m4 divert macro have an argument in order to be +dnl expanded. +define(`divert', `ifelse(`$#', `0', ``$0'', `builtin(`$0', $@)')') + + +divert(`0')dnl Output with m4 again +]}])dnl End of ifdef over the whole file. diff --git a/localize.mk b/localize.mk new file mode 100644 index 0000000..3bcf16d --- /dev/null +++ b/localize.mk @@ -0,0 +1,23 @@ +# Copyright (C) 2023 The Meme Factory, Inc. www.karlpinc.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Prefix a list of paths with "./". +# +# Syntax: +# localize(list) +# +# list A space-separated list of paths + +localize = $(patsubst %,./%,$(1)) diff --git a/rwildcard.mk b/rwildcard.mk new file mode 100644 index 0000000..f56a5e5 --- /dev/null +++ b/rwildcard.mk @@ -0,0 +1,28 @@ +# Copyright (C) 2023 The Meme Factory, Inc. www.karlpinc.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# A recursive wildcard function for make +# +# Syntax: +# rwildcard(dir,pattern) +# +# dir The directory to search. Must end in a "/". +# pattern The pattern to search for. +# +# Adapted from: +# https://stackoverflow.com/questions/4036191/sources-from-subdirectories-in-makefile + +rwildcard = $(wildcard $(1)$(2)) \ + $(foreach d,$(wildcard $(1)*),$(call rwildcard,$(d)/,$(2))) -- 2.34.1