From 7d005b259b43a788a64089f037d553c59d123df7 Mon Sep 17 00:00:00 2001
From: "Karl O. Pinc" <kop@karlpinc.com>
Date: Wed, 4 Oct 2023 21:19:22 -0500
Subject: [PATCH] Much improve the build system's performance
Have make cache the creation of targets that:
Put things in the database.
No point in connecting to the db all the time when
we don't need to.
Create directories.
There are separate targets for directory creation dependencies
so that adding to/deleting from directories themselves don't
change the dependent timestamp.
Create complex targets.
Some targets build things that are too complex to easily
test, in a later make run, that the target has been successfully
completed.
In all the above cases the build system used to go ahead and do the
extra work. But now, gen_comments.py takes a long time to run
and it's really annoying to wait all the time.
---
.gitignore | 4 +
db/schemas/Makefile | 133 +++++++++++++++++++++++++---------
make_files/make_db.mk | 75 ++++++++++++++-----
make_files/make_docs.mk | 95 ++++++++++++++++++++----
make_files/secondexpansion.mk | 10 +--
5 files changed, 244 insertions(+), 73 deletions(-)
diff --git a/.gitignore b/.gitignore
index db02425..4c24851 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,10 @@ db/set_timezone.sh
db/az_server_info.new
db/pg_settings.new
+# Made-with-make content fuzzy cache
+db/**/db_cache/
+doc/doc_cache/
+
# Built documentation
doc/sphinx-doc/
diff --git a/db/schemas/Makefile b/db/schemas/Makefile
index c513e08..99d37d7 100644
--- a/db/schemas/Makefile
+++ b/db/schemas/Makefile
@@ -87,6 +87,13 @@ SCHEMA_DIRS := tables triggers indexes functions views comments
COMMENT_DIRS := ../../doc/sphinx-doc/comment_src \
../../doc/sphinx-doc/comment_txt
+# The directory recording database create targes
+DB_CACHE := db_cache
+# We can't have $(DB_CACHE) as a prerequsite, since its timestamp
+# gets updated when the directory content is created. So Have
+# separate prerequsite that keeps track of cache creation time.
+DB_CACHE_PREREQ := $(DB_CACHE)/creationtimefile
+
# Derived variables
# (These shouldn't need editing.)
# (The use of $(ORDER) and filtering is to ensure the correct ordering.)
@@ -189,9 +196,15 @@ check:
##
# Makefile related phony targets.
+## clean-db-cache
+## Remove all record of database modifications
+.PHONY: clean-db-cache
+clean-db-cache:
+ rm -rf $(DB_CACHE)
+
## clean-db Clean up the files produced by make
.PHONY: clean-db
-clean-db:
+clean-db: clean-db-cache
rm -rf $(TARGETS)
for schema in $(ORDER) ; do \
for subdir in $(SCHEMA_DIRS) ; do \
@@ -204,14 +217,17 @@ clean-db:
done ; \
done
-# The phony targets that alter the database
+# The cached targets that alter the database
## installtables Install the tables into the database
-.PHONY: installtables
-installtables: $(PSQL_DEPENDS) createtables.sql
+installtables: $(DB_CACHE)/installtables
+
+$(DB_CACHE)/installtables: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ createtables.sql
( $(PSQL_SETUP) \
cat createtables.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ touch $(DB_CACHE)/installtables
## droptables Remove the tables from the database
.PHONY: droptables
@@ -219,22 +235,35 @@ droptables: $(PSQL_DEPENDS) droptables.sql
( $(PSQL_SETUP) \
cat droptables.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ rm -f $(DB_CACHE)/installtables \
+ $(DB_CACHE)/installindexes \
+ $(DB_CACHE)/installtriggers \
+ $(DB_CACHE)/installviews \
+ $(DB_CACHE)/installcomments
## installtriggers Install the triggers into the database
-.PHONY: installtriggers
-installtriggers: $(PSQL_DEPENDS) createtriggers.sql
+installtriggers: $(DB_CACHE)/installtriggers
+
+$(DB_CACHE)/installtriggers: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ createtriggers.sql
( $(PSQL_SETUP) \
cat createtriggers.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ touch $(DB_CACHE)/installtriggers \
+ $(DB_CACHE)/installtriggersnolock
## installtriggersnolock
## Install the triggers into the database outside of
## a transaction.
-.PHONY: installtriggersnolock
-installtriggersnolock: $(PSQL_DEPENDS) createtriggers.sql
+installtriggersnolock: $(DB_CACHE)/installtriggers
+
+$(DB_CACHE)/installtriggersnolock: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ createtriggers.sql
( $(PSQL_SETUP) \
cat createtriggers.sql ; ) \
| psql $(PSQL_ARGS)
+ touch $(DB_CACHE)/installtriggers \
+ $(DB_CACHE)/installtriggersnolock
## droptriggersnolock
## Drop the triggers from the database outside of
@@ -244,6 +273,8 @@ droptriggersnolock: $(PSQL_DEPENDS) droptriggers.sql
( $(PSQL_SETUP) \
cat droptriggers.sql ; ) \
| psql $(PSQL_ARGS)
+ rm -f $(DB_CACHE)/installtriggers \
+ $(DB_CACHE)/installtriggersnolock
## droptriggers Remove the triggers from the database
.PHONY: droptriggers
@@ -251,13 +282,18 @@ droptriggers: $(PSQL_DEPENDS) droptriggers.sql
( $(PSQL_SETUP) \
cat droptriggers.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ rm -f $(DB_CACHE)/installtriggers \
+ $(DB_CACHE)/installtriggersnolock
## installfunctions Install the functions into the database
-.PHONY: installfunctions
-installfunctions: $(PSQL_DEPENDS) createfunctions.sql
+installfunctions: $(DB_CACHE)/installfunctions
+
+$(DB_CACHE)/installfunctions: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ createfunctions.sql
( $(PSQL_SETUP) \
cat createfunctions.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ touch $(DB_CACHE)/installfunctions
## dropfunctions Remove the functions from the database
.PHONY: dropfunctions
@@ -265,13 +301,18 @@ dropfunctions: $(PSQL_DEPENDS) dropfunctions.sql
( $(PSQL_SETUP) \
cat dropfunctions.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ rm -f $(DB_CACHE)/installfunctions \
+ $(DB_CACHE)/installcomments
## installviews Install the views into the database
-.PHONY: installviews
-installviews: $(PSQL_DEPENDS) createviews.sql
+installviews: $(DB_CACHE)/installviews
+
+$(DB_CACHE)/installviews: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ createviews.sql
( $(PSQL_SETUP) \
cat createviews.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ touch $(DB_CACHE)/installviews
## dropviews Remove the views from the database
.PHONY: dropviews
@@ -279,13 +320,18 @@ dropviews: $(PSQL_DEPENDS) dropviews.sql
( $(PSQL_SETUP) \
cat dropviews.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ rm -f $(DB_CACHE)/installviews \
+ $(DB_CACHE)/installcomments
## installindexes Install the indexes into the database
-.PHONY: installindexes
-installindexes: $(PSQL_DEPENDS) createindexes.sql
+installindexes: $(DB_CACHE)/installindexes
+
+$(DB_CACHE)/installindexes: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ createindexes.sql
( $(PSQL_SETUP) \
cat createindexes.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ touch $(DB_CACHE)/installindexes
## dropindexes Remove the indexes from the database
.PHONY: dropindexes
@@ -293,13 +339,17 @@ dropindexes: $(PSQL_DEPENDS) dropindexes.sql
( $(PSQL_SETUP) \
cat dropindexes.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ rm -f $(DB_CACHE)/installindexes
## installcomments Install the comments into the database
-.PHONY: installcomments
-installcomments: $(PSQL_DEPENDS) comment_on.sql
+installcomments: $(DB_CACHE)/installcomments
+
+$(DB_CACHE)/installcomments: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ comment_on.sql
( $(PSQL_SETUP) \
cat comment_on.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ touch $(DB_CACHE)/installcomments
## dropcomments Remove the comments from the database
.PHONY: dropcomments
@@ -307,6 +357,7 @@ dropcomments: $(PSQL_DEPENDS) comment_off.sql
( $(PSQL_SETUP) \
cat comment_off.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS)
+ rm -f $(DB_CACHE)/installcomments
## install_schema_tables
## Install the tables into the schema of the TARGET_SCHEMA
@@ -528,40 +579,52 @@ $(TABLE_DROP_FILES):
#
# The SQL files that work with comments.
-comment_on.sql comment_off.sql: build_comment_files
+# (Not exactly a cache of db content updates, but I don't care.)
+comment_on.sql comment_off.sql: $(DB_CACHE)/build_comment_files
# The directories where comment sql goes
-$(COMMENT_SCHEMA_PATHS) $(COMMENT_DIRS):
+# The directories used to transform RST to text
+# The build cache
+$(COMMENT_SCHEMA_PATHS) $(COMMENT_DIRS) $(DB_CACHE):
mkdir -p $@
-# Having the template directory as a prerequsite not only creates
-# the template directory, but also creates it's content.
-.PHONY: $(COMMENT_TEMPLATE_PATHS)
-$(COMMENT_TEMPLATE_PATHS): $(PSQL_DEPENDS)
- mkdir -p $@
- ( $(PSQL_SETUP) \
- printf "SELECT * FROM gen_func_comment_tmpl('%s');\n" \
- $(patsubst %/,%, $(dir $(patsubst %/,%, $(dir $@)))) ; ) \
- | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y \
- | ./gen_func_comment_tmpls.py $@
+# Create the $(DB_CACHE) directory and setup a pre-requsite to know
+# that it's been created.
+$(DB_CACHE_PREREQ): $(DB_CACHE)
+ [ -f $(DB_CACHE_PREREQ) ] || touch $(DB_CACHE_PREREQ)
# Put the sql function we use for function comments into the db
-.PHONY: gen_func_comment_tmpl
-gen_func_comment_tmpl: $(PSQL_DEPENDS) gen_func_comment_tmpl.sql
+$(DB_CACHE)/gen_func_comment_tmpl: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ gen_func_comment_tmpl.sql
( $(PSQL_SETUP) \
cat gen_func_comment_tmpl.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y
+ touch $(DB_CACHE)/gen_func_comment_tmpl
# Builds $(COMMENT_ON_FILES) and $(COMMENT_OFF_FILES) and comment_on.sql
# and comment_off.sql.
# Caller ensures that we're calling because of changed pre-requsites.
# and creates the pre-requsites.
# Build our internal dependencies by way of prerequsites.
-.PHONY: build_comment_files
-build_comment_files: $(PSQL_DEPENDS) \
- gen_func_comment_tmpl $(COMMENT_TEMPLATE_PATHS) \
+$(DB_CACHE)/build_comment_files: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ $(DB_CACHE)/gen_func_comment_tmpl \
+ $(HTML_RST_FILES) \
$(COMMENT_SCHEMA_PATHS) $(COMMENT_DIRS)
+ for tmpl_dir in $(COMMENT_TEMPLATE_PATHS) ; do \
+ mkdir -p $${tmpl_dir} ; \
+ ( $(PSQL_SETUP) \
+ printf "SELECT * FROM gen_func_comment_tmpl('%s');\n" \
+ $(patsubst %/,%, \
+ $(dir $(patsubst %/,%, $(dir $${tmpl_dir})))) ; ) \
+ | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y \
+ | ./gen_func_comment_tmpls.py $${tmpl_dir} ; \
+ done
+ #
+ # Running gen_comments.py may take quite some time
./gen_comments.py $(HTML_SRCDIR) $(ORDER)
# Aggregate comments in schemas into 2 single files.
- ./create_sql.sh '$(COMMENT_ON_FILES)' > comment_on.sql
- ./create_sql.sh '$(COMMENT_OFF_FILES)' > comment_off.sql
+ touch $(DB_CACHE)/build_comment_files
+ { ./create_sql.sh '$(COMMENT_ON_FILES)' > comment_on.sql ; \
+ ./create_sql.sh '$(COMMENT_OFF_FILES)' > comment_off.sql ; } \
+ || { rm -f $(DB_CACHE)/build_comment_files ; \
+ exit 1 ; }
diff --git a/make_files/make_db.mk b/make_files/make_db.mk
index ae63110..5a9f4d7 100644
--- a/make_files/make_db.mk
+++ b/make_files/make_db.mk
@@ -119,14 +119,23 @@ CREATE_SUB_DB_TARGETS := db/schemas/createschemas.sql \
db/schemas/createviews.sql \
db/schemas/createtriggers.sql \
db/schemas/comment_on.sql
-DROP_SUB_DB_TARGETS := db/schemas/comment_off.sql \
- db/schemas/droptriggers.sql \
+# Don't target db/schemas/comment_off.sql since all the comments may not
+# be in the db.
+DROP_SUB_DB_TARGETS := db/schemas/droptriggers.sql \
db/schemas/dropviews.sql \
db/schemas/dropindexes.sql \
db/schemas/dropfunctions.sql \
db/schemas/droptables.sql \
db/schemas/dropschemas.sql
+# The cache directory which records when something has been
+# installed in the db.
+DB_CACHE := $(DB_DIR)/db_cache
+# We can't have $(DB_CACHE) as a prerequsite, since its timestamp
+# gets updated when the directory content is created. So Have
+# separate prerequsite that keeps track of cache creation time.
+DB_CACHE_PREREQ := $(DB_CACHE)/creationtimefile
+
## check Validate input files against expected generated output
.PHONY: check
check:
@@ -147,10 +156,12 @@ do-install-db: db/schemas/createschemas.sql \
## reinstall-db Drop all the database objects and re-install them
## This is done in two transactions, one to drop
## database objects and another to create them.
-## (This does not exercise destroy-db.)
+## (This does _not_ exercise destroy-db. It is slower
+## but more through than recreate-db.)
# This drops entire schemas to delete the database objects.
.PHONY: reinstall-db
reinstall-db: db/schemas/dropschemas.sql \
+ clean-db-cache \
$(PSQL_DEPENDS) \
$(CREATE_DB_TARGETS) $(CREATE_SUB_DB_TARGETS)
( $(PSQL_SETUP) \
@@ -163,6 +174,13 @@ reinstall-db: db/schemas/dropschemas.sql \
cat $(CREATE_DB_TARGETS) $(CREATE_SUB_DB_TARGETS) ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y
+## recreate-db Drop all the database objects and re-install them
+## This is done in two transactions, one to drop
+## database objects and another to create them.
+## (This _does_ exercise destroy-db.)
+.PHONY: recreate-db
+recreate-db: destroy-db install-db
+
## destroy-db Drop all the database objects, individually
# This drops each individual database object, then the schemas.
# Because $? is used, and we've "extra" dependencies, we need a level of
@@ -303,9 +321,20 @@ recomment: $(PSQL_DEPENDS) \
# createtypes.sql ; ) \
# | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y
+## clean-db-cache Remove all record of database modifications.
+## To reduce the need to unnecessarily connect to the
+## database, the build system tracks what has been
+## installed into the db.
+# This is an imperfect process that may still result in unnecessary
+# db connections.
+.PHONY: clean-db-cache
+clean-db-cache:
+ $(MAKE) -C db/schemas clean-db-cache
+ rm -rf $(DB_CACHE)
+
## clean-db Remove all the generated files
.PHONY: clean-db
-clean-db:
+clean-db: clean-db-cache
$(MAKE) -C db/schemas clean-db
rm -rf $(DB_TARGETS) db/creategroups.sql
@@ -326,11 +355,20 @@ sql_file: $(PSQL_DEPENDS)
$(MAKE) -C db/schemas sql_file
## install_schemas Create the schemas
-.PHONY: install_schemas
-install_schemas: $(PSQL_DEPENDS) db/schemas/createschemas.sql
+install_schemas: $(DB_CACHE)/install_schemas
+
+$(DB_CACHE):
+ mkdir -p $@
+
+$(DB_CACHE_PREREQ): $(DB_CACHE)
+ [ -f $(DB_CACHE_PREREQ) ] || touch $(DB_CACHE_PREREQ)
+
+$(DB_CACHE)/install_schemas: $(PSQL_DEPENDS) $(DB_CACHE_PREREQ) \
+ db/schemas/createschemas.sql
( $(PSQL_SETUP) \
cat db/schemas/createschemas.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y
+ touch $(DB_CACHE)/install_schemas
## destroy_schemas Drop the schemas
.PHONY: destroy_schemas
@@ -338,6 +376,7 @@ destroy_schemas: $(PSQL_DEPENDS) db/schemas/dropschemas.sql
( $(PSQL_SETUP) \
cat db/schemas/dropschemas.sql ; ) \
| psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y
+ rm -f $(DB_CACHE)/install_schemas
## install_schema_tables
## Install the tables into the schema of the TARGET_SCHEMA
@@ -430,17 +469,13 @@ db/schemas/createindexes.sql db/schemas/dropindexes.sql: $(PSQL_DEPENDS)
## db/schemas/comment_off.sql
## Build a file of sql that removes comments from all
## objects
-# Phony targets because we rely on sub-make for dependencies.
-# It would be nice to depend on db/schemas/createtables.sql etc.,
-# because those will get updated timestamps if any objects change
-# that would be a way to check for new or changed objects.
-# But that saves no work for 2 reasons, the sub-make calls a sub-make
-# and always runs, relying on the sub-make for dependencies.
-# And schemas/Makefile makes the files as a dependency; this would
-# have to be changed to a rule that calls make on itself.
-.PHONY: db/schemas/comment_on.sql db/schemas/comment_off.sql
+# In this case, the sub-make relies on us to fufull all dependencies
+# that it cannot build, as well as determine if the comments need
+# re-building. We look only at the documentation source to determine
+# when things need re-building, not the source for the actual
+# objects that are commented. That seems good enough.
db/schemas/comment_on.sql \
-db/schemas/comment_off.sql: $(HTML_RST_FILES) $(PSQL_DEPENDS) \
+db/schemas/comment_off.sql: $(HTML_PREREQS) $(PSQL_DEPENDS) \
$(TEXT_EPILOG) install_schemas
$(MAKE) -C db/schemas $(notdir $@)
@@ -455,7 +490,7 @@ $(DB_TARGETS) db/creategroups.sql: %.sql: %.m4 $(DB_DEPENDS)
## db/schemas/gen_func_comment_tmpl.sql
## Build a file of sql to make a helper function
## so the build process can find functions to comment
-db/schemas/gen_func_comment_tmpl.sql:
+db/schemas/gen_func_comment_tmpl.sql: $(DB_CACHE)/install_schemas
$(MAKE) -C db/schemas gen_func_comment_tmpl.sql
# It'd be nice to be able to empty the db of all data, but there's
@@ -480,6 +515,12 @@ db/schemas/gen_func_comment_tmpl.sql:
## I.e. you can't create an index or a trigger on a non-existant table.
##
## Debugging Tips:
+## The caching of what has been installed into the database is imperfect,
+## particuarly when more than one repository is operating against the
+## same database. If installing into the db fails this may be due
+## to inconsistencies between the cache and the db content. Try
+## `make clean-db-cache` or even `make clean` to resolve the problem.
+##
## Examining the generated SQL can be useful when debugging.
## Especially to match up line numbers reported in error messages with
## the line numbers of the m4 source. Note that reported line numbers
diff --git a/make_files/make_docs.mk b/make_files/make_docs.mk
index 3d2216f..b63fbd9 100644
--- a/make_files/make_docs.mk
+++ b/make_files/make_docs.mk
@@ -56,9 +56,6 @@ HTML_TARGETDIR:= $(HTML_TARGET)
endif
BUILDERNAME := dirhtml
endif
-# Exported because db/schemas/Makefile needs it to find the comments
-# in the docs.
-export HTML_SRCDIR
#
# Variables
@@ -86,18 +83,33 @@ SPHINX_BUILT_LATEX_LETTER := $(SPHINX_BUILDDIR)/latex_letter
SPHINX_BUILT_HTML := $(SPHINX_BUILDDIR)/$(HTML_TARGETDIR)
SPHINX_BUILT_HTML_SINGLE := $(SPHINX_BUILDDIR)/$(HTML_SINGLE_TARGET)
+# The cache directory which records when something has been
+# installed in the db.
+DOC_CACHE := $(DOCDIR)/doc_cache
+# We can't have $(DOC_CACHE) as a prerequsite, since its timestamp
+# gets updated when the directory content is created. So Have
+# separate prerequsite that keeps track of cache creation time.
+DOC_CACHE_PREREQ := $(DOC_CACHE)/creationtimefile
+
# Locations of source files, original m4 and generated
# m4 dirs
STOCK_SRCDIR := $(DOCDIR)/src
HTML_STATIC_DIR := $(DOCDIR)/_static
# Dirs of generated files
HTML_SRCDIR := $(SPHINX_DOCDIR)/$(HTML_SRCDIR)
+HTML_SRCDIR_PREREQ := $(DOC_CACHE)/html_srcdir_$(SDB_HTML_FMT)
HTML_IMAGEDIR := $(HTML_SRCDIR)/images
LATEX_SRCDIR := $(SPHINX_DOCDIR)/latex_src
+LATEX_SRCDIR_PREREQ := $(DOC_CACHE)/latex_src
LATEX_IMAGEDIR := $(LATEX_SRCDIR)/images
+LATEX_IMAGEDIR_PREREQ := $(DOC_CACHE)/latex_imagedir
VIEW_DIR := $(SPHINX_DOCDIR)/views
+VIEW_DIR_PREREQ := $(DOC_CACHE)/view_dir
HTML_VIEW_DIR := $(HTML_SRCDIR)/views
LATEX_VIEW_DIR := $(LATEX_SRCDIR)/views
+# Exported because db/schemas/Makefile needs it to find the comments
+# in the docs.
+export HTML_SRCDIR
# Locations of m4 macros
M4_DOC_INCLUDE_PATH := doc/include
@@ -131,6 +143,8 @@ SPHINXBUILD := $(SPHINX_VENV)/bin/sphinx-build -n -c $(DOCDIR)
M4_BASEDIR := $(STOCK_SRCDIR)
# The m4 files that produce rst files
+# Exported because db/schemas/Makefile needs it to find the comments
+# in the docs.
M4_FILES := $(call localize,$(call rwildcard,$(M4_BASEDIR)/,*.m4))
M4_FILES := $(filter-out %epilog.inc.m4, $(M4_FILES))
@@ -143,16 +157,19 @@ OLD_HTML_FILES := $(call localize,$(call rwildcard,$(HTML_SRCDIR)/,*))
OLD_LATEX_FILES := $(call localize,$(call rwildcard,$(LATEX_SRCDIR)/,*))
# The html files to build and the directories that contain the files.
-HTML_RST_FILES := \
+# (Used by db/schemas/Makefile to tell if comments should be re-made.)
+export HTML_RST_FILES := \
$(patsubst %.m4,%.rst,\
$(subst /$(STOCK_SRCDIR)/,/$(HTML_SRCDIR)/,$(M4_FILES)))
HTML_DIRS := $(sort $(dir $(HTML_RST_FILES)))
+HTML_DIRS_PREREQ := $(DOC_CACHE)/html_dirs_$(SDB_HTML_FMT)
# The latex files to build and the directories that contain the files
LATEX_RST_FILES := \
$(patsubst %.m4,%.rst,\
$(subst /$(STOCK_SRCDIR)/,/$(LATEX_SRCDIR)/,$(M4_FILES)))
LATEX_DIRS := $(sort $(dir $(LATEX_RST_FILES)))
+LATEX_DIRS_PREREQ := $(DOC_CACHE)/latex_dirs
# The epilog
SOURCE_EPILOG := $(M4_BASEDIR)/epilog.inc.m4
@@ -168,6 +185,7 @@ SPHINX_PDF_LETTER_TARGET := $(SPHINX_BUILT_LATEX_LETTER)/$(PDF_NAME)
# Image targets
INKSCAPE_BASEDIR := $(DOCDIR)/diagrams/
IMAGES_BASEDIR := $(SPHINX_DOCDIR)/images/
+IMAGES_BASEDIR_PREREQ := $(DOC_CACHE)/images_basedir
INKSCAPE_FILES := $(wildcard $(INKSCAPE_BASEDIR)*.svg)
PARTIAL_PATHS := $(subst $(INKSCAPE_BASEDIR),$(IMAGES_BASEDIR), \
@@ -196,8 +214,8 @@ HTML_PREREQS := install-sphinx html_rst $(HTML_IMAGE_FILES) $(HTML_EPILOG) \
$(HTML_STATIC_DIR) $(HTML_IMAGEDIR) $(HTML_VIEW_DIR)
# Prerequsites for latex (pdf) generation
-LATEX_PREREQS := latex_rst $(PNG_FILES) $(LATEX_EPILOG) $(LATEX_IMAGEDIR) \
- $(LATEX_VIEW_DIR)
+LATEX_PREREQS := latex_rst $(PNG_FILES) $(LATEX_EPILOG) \
+ $(LATEX_IMAGEDIR_PREREQ) $(LATEX_VIEW_DIR)
##
## The available targets for make (make TARGET) are:
@@ -223,9 +241,19 @@ html-single: $(HTML_PREREQS)
.PHONY: pdf
pdf: install-sphinx $(PDF_A4_TARGET) $(PDF_LETTER_TARGET)
+## clean-docs-cache Remove all record of documentation modifications.
+## To reduce the need to unnecessarily rebuild docs,
+## the build system tracks what has been built
+## with a cache.
+# This is an imperfect process that may still result in unnecessary
+# rebuilding.
+.PHONY: clean-docs-cache
+clean-docs-cache:
+ rm -rf $(DOC_CACHE)
+
## mostlyclean-docs Delete all the generated files but the virtualenv
.PHONY: mostlyclean-docs
-mostlyclean-docs:
+mostlyclean-docs: clean-docs-cache
rm -rf $(CLEAN_DOC_TARGETS)
## clean-docs Delete all user generated files
@@ -272,8 +300,11 @@ install-html-single: html-single $(DOC_ROOT)
$(SPHINX_BUILT_HTML_SINGLE)/ $(HTML_SINGLE_PATH)
## install-sphinx Install sphinx in a virtualenv
-.PHONY: install-sphinx
-install-sphinx: sphinx-venv $(SPHINX_VENV)/bin/sphinx-build
+install-sphinx: $(DOC_CACHE)/install-sphinx
+
+$(DOC_CACHE)/install-sphinx: $(DOC_CACHE_PREREQ) sphinx-venv \
+ $(SPHINX_VENV)/bin/sphinx-build
+ touch $(DOC_CACHE)/install-sphinx
## report-old Report on outdated packages in the sphinx venv
.PHONY: report-old
@@ -281,8 +312,9 @@ report-old:
$(SPHINX_VENV)/bin/pip list -o
## sphinx-venv Make a virtual environment for sphinx to run from
-.PHONY: sphinx-venv
-sphinx-venv: $(SPHINX_VENV)
+sphinx-venv: $(DOC_CACHE)/sphinx-venv
+
+$(DOC_CACHE)/sphinx-venv: $(DOC_CACHE_PREREQ) $(SPHINX_VENV)
$(SPHINX_VENV):
python3 -m venv $(SPHINX_VENV)
@@ -305,8 +337,11 @@ $(PDF_LETTER_TARGET): $(SPHINX_PDF_LETTER_TARGET)
cp $(SPHINX_PDF_LETTER_TARGET) $(PDF_LETTER_TARGET)
## html_rst Build RST files from m4 source for HTML generation
-.PHONY: html_rst
-html_rst: $(HTML_DIRS) $(HTML_RST_FILES) $(HTML_SRCDIR)
+html_rst: $(DOC_CACHE)/html_rst_$(SDB_HTML_FMT)
+
+$(DOC_CACHE)/html_rst_$(SDB_HTML_FMT): $(DOC_CACHE_PREREQ) \
+ $(HTML_DIRS_PREREQ) \
+ $(HTML_RST_FILES) $(HTML_SRCDIR_PREREQ)
# Get rid of anything old not a *.rst file, and not in the source
rm -rf \
$(filter-out \
@@ -318,10 +353,11 @@ html_rst: $(HTML_DIRS) $(HTML_RST_FILES) $(HTML_SRCDIR)
rsync -a --exclude='*.m4' --exclude='*$(EMACS_BK_CHR)' \
$(STOCK_SRCDIR)/ \
$(HTML_SRCDIR)
+ touch $(DOC_CACHE)/html_rst_$(SDB_HTML_FMT)
## latex_rst Build RST files from m4 source for LaTeX generation
.PHONY: latex_rst
-latex_rst: $(LATEX_DIRS) $(LATEX_RST_FILES) $(LATEX_SRCDIR)
+latex_rst: $(LATEX_DIRS_PREREQ) $(LATEX_RST_FILES) $(LATEX_SRCDIR_PREREQ)
# Get rid of anything old not a *.rst file, and not in the source
rm -rf $(filter-out \
$(LATEX_RST_FILES) \
@@ -362,10 +398,37 @@ $(PDF_LETTER_PATH): $(PDF_LETTER_TARGET)
rsync -a $(PDF_LETTER_TARGET) $(PDF_LETTER_PATH)
# Construct all the source dirs used to build output
+# When it comes to making directores, we have a cached target
+# in $(DOC_CACHE) to see if the directory(ies) have been made.
+# We always run the mkdir, but the point of having the prerequsite
+# target is to keep a fixed timestamp of when the directory was
+# created. Otherwise, just depending on the directory means that
+# the directory's timestamp updates every time a file is added
+# to or removed from the directory.
$(LATEX_DIRS) $(HTML_DIRS) $(HTML_SRCDIR) $(LATEX_SRCDIR) \
- $(HTML_STATIC_DIR) $(DOC_ROOT) $(IMAGES_BASEDIR) $(VIEW_DIR):
+$(HTML_STATIC_DIR) $(DOC_ROOT) $(IMAGES_BASEDIR) $(VIEW_DIR) \
+$(DOC_CACHE):
mkdir -p $@
+$(LATEX_DIRS_PREREQ): $(DOC_CACHE_PREREQ) $(LATEX_DIRS)
+ [ -f $(LATEX_DIRS_PREREQ) ] || touch $(LATEX_DIRS_PREREQ)
+$(HTML_DIRS_PREREQ): $(DOC_CACHE_PREREQ) $(HTML_DIRS)
+ [ -f $(HTML_DIRS_PREREQ) ] || touch $(HTML_DIRS_PREREQ)
+$(HTML_SRCDIR_PREREQ): $(DOC_CACHE_PREREQ) $(HTML_SRCDIR)
+ [ -f $(HTML_SRCDIR_PREREQ) ] || touch $(HTML_SRCDIR_PREREQ)
+$(LATEX_SRCDIR_PREREQ): $(DOC_CACHE_PREREQ) $(LATEX_SRCDIR)
+ [ -f $(LATEX_SRCDIR_PREREQ ] || touch $(LATEX_SRCDIR_PREREQ)
+$(LATEX_IMAGEDIR_PREREQ): $(DOC_CACHE_PREREQ) $(LATEX_IMAGEDIR)
+ [ -f $(LATEX_IMAGEDIR_PREREQ ] || touch $(LATEX_IMAGEDIR_PREREQ)
+$(IMAGES_DIR_PREREQ): $(DOC_CACHE_PREREQ) $(IMAGES_DIR)
+ [ -f $(IMAGES_DIR_PREREQ) ] || touch $(IMAGES_DIR_PREREQ)
+$(IMAGES_BASEDIR_PREREQ): $(DOC_CACHE_PREREQ) $(IMAGES_BASEDIR)
+ [ -f $(IMAGES_BASEDIR_PREREQ) ] || touch $(IMAGES_BASEDIR_PREREQ)
+$(VIEW_DIR_PREREQ): $(DOC_CACHE_PREREQ) $(VIEW_DIR)
+ [ -f $(VIEW_DIR_PREREQ) ] || touch $(VIEW_DIR_PREREQ)
+$(DOC_CACHE_PREREQ): $(DOC_CACHE)
+ [ -f $(DOC_CACHE_PREREQ) ] || touch $(DOC_CACHE_PREREQ)
+
# Sphinx requires all images be located in the $(SPHINX_DOCDIR),
# at least if the paths are not relative to the source RST file and
# we don't want relative paths. So symlink the image dir into
@@ -375,7 +438,7 @@ $(HTML_IMAGEDIR) $(LATEX_IMAGEDIR):
# Likewise with the SQL of view bodies
$(HTML_VIEW_DIR) $(LATEX_VIEW_DIR):
- ln -s ../views $@
+ ln -sf ../views $@
#
# The files sphinx builds for a pdf
diff --git a/make_files/secondexpansion.mk b/make_files/secondexpansion.mk
index 0e9612f..c4f1bd6 100644
--- a/make_files/secondexpansion.mk
+++ b/make_files/secondexpansion.mk
@@ -21,19 +21,19 @@
# Construct *.svg files
$(SVG_FILES): $$(patsubst %.svg,%.svg,\
$$(subst $(IMAGES_BASEDIR),$(INKSCAPE_BASEDIR),$$@))\
- $(DOC_DEPENDS) $(IMAGES_BASEDIR)
+ $(DOC_DEPENDS) $(IMAGES_BASEDIR_PREREQ)
inkscape --export-filename=$@ --export-plain-svg $<
# Construct *.png files
$(PNG_FILES): $$(patsubst %.png,%.svg,\
$$(subst $(IMAGES_BASEDIR),$(INKSCAPE_BASEDIR),$$@))\
- $(DOC_DEPENDS) $(IMAGES_BASEDIR)
+ $(DOC_DEPENDS) $(IMAGES_BASEDIR_PREREQ)
inkscape --export-filename=$@ $<
# Build all the RST for html output
$(HTML_RST_FILES): $$(patsubst %.rst,%.m4,\
$$(subst $(HTML_SRCDIR),$(STOCK_SRCDIR),$$@)) \
- $(HTML_DIRS) $(DOC_DEPENDS)
+ $(HTML_DIRS_PREREQ) $(DOC_DEPENDS)
m4 $(M4_DOC_INCLUDE_ARGS) \
--define=sdb_output_fmt=$(SDB_HTML_FMT) \
$< \
@@ -42,7 +42,7 @@ $(HTML_RST_FILES): $$(patsubst %.rst,%.m4,\
# Build all the RST for latex output
$(LATEX_RST_FILES): $$(patsubst %.rst,%.m4,\
$$(subst $(LATEX_SRCDIR),$(STOCK_SRCDIR),$$@)) \
- $(LATEX_DIRS) $(DOC_DEPENDS)
+ $(LATEX_DIRS_PREREQ) $(DOC_DEPENDS)
m4 $(M4_DOC_INCLUDE_ARGS) \
--define=sdb_output_fmt=latex \
$(patsubst %.rst,%.m4,\
@@ -52,7 +52,7 @@ $(LATEX_RST_FILES): $$(patsubst %.rst,%.m4,\
# The "body" of the CREATE VIEW statements.
$(VIEW_SQL): db/schemas/$$(notdir $$(patsubst \
%/,%,$$(dir $$@)))/views/create/$$(notdir $$@) \
- $(VIEW_DIR) doc/extract_view.py
+ $(VIEW_DIR_PREREQ) doc/extract_view.py
mkdir -p $(VIEW_DIR)/$$(basename $$(dirname $@))
doc/extract_view.py \
< $< \
--
2.34.1