From f02f2b7d1bd1208be5894c4ba52501c6e7292dd3 Mon Sep 17 00:00:00 2001 From: "Karl O. Pinc" Date: Sun, 1 Oct 2023 14:13:43 -0500 Subject: [PATCH] Basics of putting comments into the db --- .gitignore | 1 + Makefile | 10 - db/schemas/Makefile | 92 ++++++++- db/schemas/gen_comments.py | 270 +++++++++++++++++++++++++++ db/schemas/gen_func_comment_tmpl.m4 | 165 ++++++++++++++++ db/schemas/gen_func_comment_tmpls.py | 74 ++++++++ make_files/make_db.mk | 48 ++++- make_files/make_docs.mk | 5 +- 8 files changed, 649 insertions(+), 16 deletions(-) create mode 100755 db/schemas/gen_comments.py create mode 100644 db/schemas/gen_func_comment_tmpl.m4 create mode 100755 db/schemas/gen_func_comment_tmpls.py diff --git a/.gitignore b/.gitignore index 339774d..db02425 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # The generated sql db/schemas/**/*.sql db/*.sql +db/schemas/*/comments/ # Generated shell scripts db/psql_setup.sh diff --git a/Makefile b/Makefile index 603a535..2fe163c 100644 --- a/Makefile +++ b/Makefile @@ -15,16 +15,6 @@ # # Karl O. Pinc -# To get a database of comments, look for: -# ^.. |TABLEORTABLECOLUMN| replace:: .* -# Save in a directory structure, each file named with TABLEORTABLECOLUMN -# and the content is ".*". -# What do we do to get rid of markup in the replacement text? -# Run it through `sphinx-build -b text ...` (ignoring link errors) -# Parse it ourselves -# Then query the db for all tables and columns (and functions), delete -# all the comments for each and insert new ones. - # # Variables # diff --git a/db/schemas/Makefile b/db/schemas/Makefile index 93cf30b..2465929 100644 --- a/db/schemas/Makefile +++ b/db/schemas/Makefile @@ -41,6 +41,7 @@ FUNCTION_SCHEMAS := sokwedb TRIGGER_SCHEMAS := lookup sokwedb VIEW_SCHEMAS := sokwedb INDEX_SCHEMAS := sokwedb +COMMENT_SCHEMAS := lookup sokwedb ## ## CAUTION: This Makefile is not designed to be run directly. It is normally @@ -56,7 +57,9 @@ TARGETS := createtables.sql droptables.sql \ createfunctions.sql dropfunctions.sql \ createviews.sql dropviews.sql \ createindexes.sql dropindexes.sql \ - createschemas.sql dropschemas.sql + createschemas.sql dropschemas.sql \ + commentson.sql commentsoff.sql \ + gen_func_comment_tmpl.sql # Paths to the m4 include directories. export M4_GLOBAL_INCLUDE_PATH := ../../$(M4_GLOBAL_INCLUDE_PATH) @@ -66,13 +69,18 @@ M4_DB_INCLUDE_ARGS := -I $(M4_DB_INCLUDE_PATH) -I $(M4_GLOBAL_INCLUDE_PATH) # psql helper script PSQL_SETUP := ../../$(PSQL_SETUP) +# Where the RST files are +HTML_SRCDIR := ../../$(HTML_SRCDIR) +# The RST files +HTML_RST_FILES := $(patsubst %,../../%, $(HTML_RST_FILES)) + # Rebuild when any of these change export DB_DEPENDS := $(patsubst %,../../%,$(DB_DEPENDS)) Makefile \ check_target_schema.sh # The directories that can exist in each schema directory, _in_ _the_ # _order_ in which they should be used to create db objects. -SCHEMA_DIRS := tables triggers indexes functions views +SCHEMA_DIRS := tables triggers indexes functions views comments # Derived variables # (These shouldn't need editing.) @@ -105,6 +113,14 @@ INDEX_CREATE_FILES := $(patsubst %,%/createthings.sql, \ $(INDEX_SCHEMA_PATHS)) INDEX_DROP_FILES := $(patsubst %,%/dropthings.sql, \ $(INDEX_SCHEMA_PATHS)) +COMMENT_SCHEMA_PATHS := $(patsubst %,%/comments, \ + $(filter $(COMMENT_SCHEMAS),$(ORDER))) +COMMENT_TEMPLATE_PATHS := $(patsubst %,%/gen_comment_tmpls, \ + $(COMMENT_SCHEMA_PATHS)) +COMMENT_ON_FILES := $(patsubst %,%/comment_on.sql, \ + $(COMMENT_SCHEMA_PATHS)) +COMMENT_OFF_FILES := $(patsubst %,%/comment_off.sql, \ + $(COMMENT_SCHEMA_PATHS)) ## check Test that an input file exists for an expected output file .PHONY: check @@ -179,6 +195,7 @@ clean-db: $(MAKE) -C $${schema}/$${subdir} clean-db \ || exit 1 ; \ fi ; \ + rm -rf $${schema}/$${subdir}/comments ; \ done ; \ done @@ -272,6 +289,20 @@ dropindexes: dropindexes.sql cat dropindexes.sql ; ) \ | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) +## installcomments Install the comments into the database +.PHONY: installcomments +installcomments: comment_on.sql + ( $(PSQL_SETUP) \ + cat comment_on.sql ; ) \ + | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) + +## dropcomments Remove the comments from the database +.PHONY: dropcomments +dropcomments: comment_off.sql + ( $(PSQL_SETUP) \ + cat comment_off.sql ; ) \ + | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) + ## install_schema_tables ## Install the tables into the schema of the TARGET_SCHEMA ## variable @@ -391,7 +422,12 @@ sql_file: ## A file with the SQL to make all schemas ## dropschemas.sql ## A file with the SQL to drop all schemas -dropschemas.sql createschemas.sql: %.sql: %.m4 $(DB_DEPENDS) +## gen_func_comment_tmpl.sql +## A file of sql which creates the helper function +## that generates SQL fragments to help maintain +## comments on functions. +dropschemas.sql createschemas.sql \ +gen_func_comment_tmpl.sql: %.sql: %.m4 $(DB_DEPENDS) m4 $(M4_DB_INCLUDE_ARGS) $< > $@ @@ -454,6 +490,14 @@ createindexes.sql: $(DB_DEPENDS) $(INDEX_CREATE_FILES) \ dropindexes.sql: $(DB_DEPENDS) $(INDEX_DROP_FILES) drop_sql.sh ./drop_sql.sh '$(INDEX_DROP_FILES)' > dropindexes.sql +## destroycomments.sql +## A file with SQL which removes all the comments +## in SokweDB's schemas +# This must depend on any sql that creates anything to which we attach +# a comment. +destroycomments.sql: $(DB_DEPENDS) commentschemas.sql createfunctions.sql \ + createtables.sql createviews.sql + # Pick up any changes in the schemas and re-generate the sub-sql files. # Always let the sub-make do it so that dependencies are picked up. @@ -473,3 +517,45 @@ $(TRIGGER_DROP_FILES) $(INDEX_DROP_FILES): .PHONY: $(TABLE_DROP_FILES) $(TABLE_DROP_FILES): $(MAKE) -C $$(dirname $@) droptables.sql + +# +# Documentation related targets +# + +# The SQL files that work with comments. +comment_on.sql comment_off.sql: build_comment_files + +# The directories where comment sql goes +$(COMMENT_SCHEMA_PATHS): + mkdir -p $@ + +# Having the template directory as a prerequsite not only creates +# the template directory, but also creates it's content. +.PHONY: $(COMMENT_TEMPLATE_PATHS) +$(COMMENT_TEMPLATE_PATHS): + mkdir -p $@ + ( $(PSQL_SETUP) \ + printf "SELECT * FROM gen_func_comment_tmpl('%s');\n" \ + $(patsubst %/,%, $(dir $(patsubst %/,%, $(dir $@)))) ; ) \ + | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y \ + | ./gen_func_comment_tmpls.py $@ + +# Put the sql function we use for function comments into the db +.PHONY: gen_func_comment_tmpl +gen_func_comment_tmpl: gen_func_comment_tmpl.sql + ( $(PSQL_SETUP) \ + cat gen_func_comment_tmpl.sql ; ) \ + | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y + +# Builds $(COMMENT_ON_FILES) and $(COMMENT_OFF_FILES) and comment_on.sql +# and comment_off.sql. +# Caller ensures that we're calling because of changed pre-requsites. +# and creates the pre-requsites. +# Build our internal dependencies by way of prerequsites. +.PHONY: build_comment_files +build_comment_files: $(COMMENT_TEMPLATE_PATHS) gen_func_comment_tmpl \ + $(COMMENT_SCHEMA_PATHS) + ./gen_comments.py $(HTML_SRCDIR) $(ORDER) + # Aggregate comments in schemas into 2 single files. + ./create_sql.sh '$(COMMENT_ON_FILES)' > comment_on.sql + ./create_sql.sh '$(COMMENT_OFF_FILES)' > comment_off.sql diff --git a/db/schemas/gen_comments.py b/db/schemas/gen_comments.py new file mode 100755 index 0000000..dbdda03 --- /dev/null +++ b/db/schemas/gen_comments.py @@ -0,0 +1,270 @@ +#!/usr/bin/python3 +# Copyright (C) 2023 The Meme Factory, Inc. http://www.karlpinc.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +"""Build sql which creates comments on database objects, and SQL that +removes the comments + +Read from stdin and write files into the OUTPUT_DIR directory +into files that hold SQL lines having identical function names. + +Syntax: gen_func_comment_tmpls.py html_srcdir schema... + +Input: + html_srcdir Directory containing RST source -- where the comments come from + schema... One or more schemas to examine for database objects + +Side Effects: + For each schema, produces {schema}/comments/comment_on.sql and + {schema}/comments/comment_off.sql files. + +Comments are established/removed when they both: + Exist as RST replacement text with a substitution reference of + OBJECTNAME_summary in the RST files. + Can be found as objects in the {schema}/{type}/create/ directories. + +When a table is found it is assumed that there are columns for the +TABLENAME.COLUMNNAME_summary substitution references. + +Remarks: + +The comments are Sphinx RST, as expanded from the m4 documentation source. +If we really wanted to we could expand them to text. (Sadly, sphinx +has no filter for this.) + +Assumes that files which create db objects have a ".m4" suffix. + +Relies on the commented database objects' names being schema-unique. +(The documenation does not schema qualify objects having comments.) + +""" +# Karl O. Pinc +# +# It would be nice to use attrs, but that means managing library +# versions. Instead, keep it simple (assign attributes in build() +# instead of __init__()) and pretend that the standard library is +# forward and backward compatible. + +import pathlib +import re +import sys + +OUTPUT_DIR_FMT = "{schema}/comments" +HEADING = "-- DO NOT EDIT this file. It was automatically generated.\n" +OBJ_TYPES = ["functions", "tables", "views"] +COMMENT_TEST = re.compile(r"\s*(?:--\s)|$") +# 3 match groups: indentation, replacement name, everything after the ":: " +OBJ_SUMMARY = re.compile( + r"(.. +)\|" + r"((?:(?:[a-z]|[A-Z]|[0-9]|_)+)" + r"(?:\.(?:(?:[a-z]|[A-Z]|[0-9]|_)+)){,1})" + r"_summary\|\s+replace::(?:\s*(.*))+" +) + +SQL = { + "tables": "COMMENT ON TABLE {name} IS ", + "column": "COMMENT ON COLUMN {name} IS ", + "views": "COMMENT ON VIEW {name} IS ", +} + + +class DuplicateError(Exception): + """Db object exists in more than one schema""" + +class NoFileWarning(Warning): + """No m4 file to go with what looks like a db comment""" + +class SQLFile: + """Write to a file of sql statements that manipulate comments""" + + def build(self, path): + """Initialize""" + self.path = path + self.fd = None + + def shutdown(self): + """Discard resources""" + if self.fd is not None: + self.fd.close() + + def open(self): + """Open the file for writing, if it's not open""" + if self.fd is None: + self.fd = self.path.open("w") + self.fd.write(HEADING) + + def write(self, stmnt): + """Write to the file""" + self.open() + self.fd.write(stmnt) + + +class CommentFiles: + """Write to all the files with SQL comment statements for a given db + object + """ + + def build(self, schema): + """Initialize""" + self.on_file = SQLFile() + cdir = pathlib.Path(schema, "comments") + self.on_file.build(cdir / "comment_on.sql") + self.off_file = SQLFile() + self.off_file.build(cdir / "comment_off.sql") + + def shutdown(self): + """Discard resources""" + self.on_file.shutdown() + self.off_file.shutdown() + + def write(self, stmnts, comment): + """Write COMMENT ON statements to the files""" + for stmnt in stmnts: + self.on_file.write(f"{stmnt}$com${comment}$com$;\n") + self.off_file.write(f"{stmnt}NULL;\n") + + +class DBObj: + """Database object specification data from the build system""" + + def build(self, objname, typedir, comment_file): + """Constructor""" + self.objtype = typedir.name + self.cf = comment_file + self.typepath = str(typedir) + + sdir = typedir.parent + if self.objtype == "functions": + self.sql = self.function_lookup(sdir, objname) + elif self.objtype == "tables" and objname.find(".") != -1: + self.sql = [SQL["column"].format(name=objname)] + else: + self.sql = [SQL[self.objtype].format(name=objname)] + + def function_lookup(self, sdir, objname): + """Return list of sql statements which comment the function""" + file = sdir / "comments" / "gen_comment_tmpls" / f"{objname}.sqlfrag" + sql = [] + with file.open("r") as fd: + for line in fd: + if COMMENT_TEST.match(line) is None: + sql.append(line[:-1]) + return sql + + +def find_schemas_objects(schema, cf, db_objs): + """Mutate db_objs{}, adding all the objects in the given schema""" + sdir = pathlib.Path(schema) + for child in sdir.iterdir(): + if child.name in OBJ_TYPES and child.is_dir(): + for file in (child / "create").iterdir(): + file_name = str(file.name) + if file.is_file() and file_name[-3:] == ".m4": + objname = file_name.lower()[:-3] + if objname in db_objs: + # Multiple functions with the same name can exist. + # No reason they have to be in the same file. + old = db_objs[objname].typepath + old_schema = old.split("/")[0] + if old_schema != schema: + raise DuplicateError( + "Cannot write comments:" + f" db object {objname} in {child}" + f" is duplicated in {old}" + ) + else: + db_obj = DBObj() + db_obj.build(objname, child, cf) + db_objs[objname] = db_obj + + +def find_objects(schemas): + """Given a list of schemas, return a dict of all the database objects + which could have comments attached. The dict is keyed by db + object name. It's values are DBObj instances. Columns are + special; we don't need separate files to store their comment's + SQL, we store them with the table comment. + """ + db_objs = {} + for schema in schemas: + cf = CommentFiles() + cf.build(schema) + find_schemas_objects(schema, cf, db_objs) + return db_objs + + +def close_objs(db_objs): + """Close all the CommentFiles instances we've saved""" + for cf in {db_obj.cf for db_obj in db_objs.values()}: + cf.shutdown() + + +def comments_from_file(file, db_objs): + """Read the file and process all comments""" + with file.open("r") as fd: + in_match = False + lineno = 1 + for line in fd: + if in_match: + if line[0:indent] == prefix: + comment.append(line[indent:]) + else: + db_obj = db_objs[obj] + db_obj.cf.write(db_obj.sql, "".join(comment)[:-1]) + in_match = False + if not in_match: + match = OBJ_SUMMARY.match(line) + if match is not None: + obj = match.group(2).lower() + if obj in db_objs: + indent = len(match.group(1)) + prefix = " " * indent + comment = [match.group(3).lstrip()] + in_match = True + else: + print(NoFileWarning( + f"Warning: File {file}: Line {lineno}:" + f" Replacement string: {obj}:" + f" Looks like a db object comment but no" + " corresponding m4 file exists")) + lineno += 1 + + +def traverse_rst(rst_dir, db_objs): + """Traverse the directory structure and process all files""" + for entry in rst_dir.iterdir(): + if entry.is_file(): + comments_from_file(entry, db_objs) + if entry.is_dir(): + traverse_rst(entry, db_objs) + + +def write_comment_sql(html_srcdir, db_objs): + """Write comments into {schema}/comments/comment_on.sql (and + comment_off.sql) + """ + traverse_rst(html_srcdir, db_objs) + close_objs(db_objs) + + +def main(): + """Standard entrypoint""" + html_srcdir = pathlib.Path(sys.argv[1]) + schemas = sys.argv[2:] + write_comment_sql(html_srcdir, find_objects(schemas)) + + +if __name__ == "__main__": + main() diff --git a/db/schemas/gen_func_comment_tmpl.m4 b/db/schemas/gen_func_comment_tmpl.m4 new file mode 100644 index 0000000..caab1a2 --- /dev/null +++ b/db/schemas/gen_func_comment_tmpl.m4 @@ -0,0 +1,165 @@ +dnl Copyright (C) 2023 The Meme Factory, Inc., http://www.karlpinc.com/ +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU Affero General Public License as published +dnl by the Free Software Foundation, either version 3 of the License, or +dnl (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU Affero General Public License for more details. +dnl +dnl You should have received a copy of the GNU Affero General Public License +dnl along with this program. If not, see . +dnl +dnl Make a plpgsql function that writes SQL fragements which manipulate +dnl comments on functions. +dnl +dnl This is necessary since the documentation does not contain enough +dnl information on function arguments (data types and other call +dnl signature information) to be able to create or remove comments +dnl on functions. +dnl +dnl Bugs: +dnl This does not really belong in the db directory, it has more to +dnl do with documentation than runtime database behavior. But having +dnl it in the db directory let's us easily use the db's macros. +dnl +dnl Uses pg_type.typname as the type signature. Ignores pg_type.typcatagory +dnl and does not attempt to add the [] syntax to the end of arrays. +dnl This is because pg_type.typname (as of PGv16) is undocumented for array +dnl types; there is no telling how to get a name value that [] can be appended +dnl to in order to generate a type signature. (Seems to work, for +dnl at least the text[] type.) +dnl +dnl Karl O. Pinc +dnl +dnl +dnl m4 includes +include(`copyright.m4') +include(`constants.m4') +include(`macros.m4') +include(`functions.m4') +dnl + +CREATE OR REPLACE FUNCTION gen_func_comment_tmpl(scema TEXT) + RETURNS SETOF TEXT + LANGUAGE plpgsql + STABLE + PARALLEL SAFE + sdb_function_set_search_path + AS $$ + + -- Return SQL fragments to manipulate the comments of all the functions + -- in a schema. + -- + -- AGPL_notice(` --', `2023', `Karl O. Pinc ') + -- + -- Syntax: gen_func_comment_tmpl(scema) + -- + -- Input: + -- scema The schema to search for functions. + -- + -- Returns: + -- A line, "COMMENT ON FUNCTION funcname (...) IS" for every function + -- call signature in schema "scema". The lines are guarenteed to be + -- ordered by the function name (placeholder "funcname" above). + -- + -- Remarks: + -- Used to maintain the database's structure (i.e. comments). Is not + -- otherwise part of the database's functionality. + -- + -- Bugs: + -- We could improve performance, likely, with a result array + -- instead of doing concatenation, but who cares? + -- We assume that type names are unique, or at least unique within + -- the current search path. + + DECLARE + stmnt TEXT; + prefix CONSTANT TEXT := 'COMMENT ON FUNCTION '; + suffix CONSTANT TEXT := E') IS '; + + this_proname pg_proc.proname%TYPE; + this_proargtypes pg_proc.proargtypes%TYPE; + this_proallargtypes pg_proc.proallargtypes%TYPE; + this_proargmodes pg_proc.proargmodes%TYPE; + this_proargnames pg_proc.proargnames%TYPE; + + this_typname pg_type.typname%TYPE; + + argtypes oid[]; + argtype oid; + separator TEXT; + pos INT; + len INT; + mode_code TEXT; + mode TEXT; + argname TEXT; + BEGIN + + FOR this_proname, this_proargtypes, this_proallargtypes + , this_proargmodes, this_proargnames + IN + SELECT pg_proc.proname, pg_proc.proargtypes, pg_proc.proallargtypes + , pg_proc.proargmodes, pg_proc.proargnames + FROM pg_proc + JOIN pg_namespace + ON (pg_namespace.oid = pg_proc.pronamespace) + WHERE pg_namespace.nspname = scema + LOOP + stmnt := prefix || this_proname || ' ('; + + separator := ''; + IF this_proallargtypes IS NULL THEN + -- COMMENT ON ignores a lot of input. It is sufficent to give it + -- only the pg_proc.progargtypes information in this case. + argtypes := this_proargtypes; + FOREACH argtype IN ARRAY argtypes LOOP + SELECT pg_type.typname + INTO this_typname + FROM pg_type + WHERE pg_type.oid = argtype; + stmnt := stmnt || separator || this_typname; + + separator := ', '; + END LOOP; + ELSE + -- A full call signature is required + pos := 1; + len := array_length(this_proallargtypes, 1); + WHILE pos <= len LOOP + SELECT pg_type.typname + INTO this_typname + FROM pg_type + WHERE pg_type.oid = this_proallargtypes[pos]; + + mode_code := this_proargmodes[pos]; + mode := CASE + WHEN mode_code = 'i' THEN 'IN ' + WHEN mode_code = 'o' THEN 'OUT ' + WHEN mode_code = 'b' THEN 'INOUT ' + WHEN mode_code = 'v' THEN 'VARIADIC ' + WHEN mode_code = 't' THEN 'TABLE ' + END; + + IF this_proargnames IS NULL THEN + argname := ''; + ELSE + argname := this_proargnames[pos] || ' '; + END IF; + + stmnt := stmnt || separator || mode || argname || this_typname; + + separator := ', '; + pos := pos + 1; + END LOOP; + END IF; + + RETURN NEXT stmnt || suffix; + END LOOP; + + RETURN; + END; +$$; diff --git a/db/schemas/gen_func_comment_tmpls.py b/db/schemas/gen_func_comment_tmpls.py new file mode 100755 index 0000000..a9daebf --- /dev/null +++ b/db/schemas/gen_func_comment_tmpls.py @@ -0,0 +1,74 @@ +#!/usr/bin/python3 +# Copyright (C) 2023 The Meme Factory, Inc. http://www.karlpinc.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +"""Read from stdin and write files into the OUTPUT_DIR directory +into files that hold SQL lines having identical function names. + +Syntax: gen_func_comment_tmpls.py output_dir + +Input: + output_dir The directory in which to place a file for each function input +""" +# Karl O. Pinc +# +import pathlib +import sys + +OUTPUT_DIR = pathlib.Path(sys.argv[1]) +FNAME_FMT = "{obj}.sqlfrag" +HEADING = "-- DO NOT EDIT this file. It was automatically generated.\n" + + +def write_file(fname, content): + """Write the content into a file""" + if content != []: + target_path = OUTPUT_DIR / FNAME_FMT.format(obj=fname) + with target_path.open("w") as fd: + fd.write(HEADING) + for line in content: + fd.write(line) + + +def get_fname(line): + """Return the name of the function mentioned in the line""" + return line.split(" ")[3] + + +def write_files(fd): + """Write a file per function""" + content = [] + fname = "" + for line in fd: + line = line.lstrip() + if line != "": + this_fname = get_fname(line) + if this_fname == fname: + content.append(line) + else: + write_file(fname, content) + content = [line] + fname = this_fname + write_file(fname, content) + + +def main(): + """Standard entrypoint""" + with open("/dev/stdin", "r") as fd: + write_files(fd) + + +if __name__ == "__main__": + main() diff --git a/make_files/make_db.mk b/make_files/make_db.mk index 6a430a1..2eaaea7 100644 --- a/make_files/make_db.mk +++ b/make_files/make_db.mk @@ -111,8 +111,10 @@ CREATE_SUB_DB_TARGETS := db/schemas/createschemas.sql \ db/schemas/createindexes.sql \ db/schemas/createfunctions.sql \ db/schemas/createviews.sql \ - db/schemas/createtriggers.sql -DROP_SUB_DB_TARGETS := db/schemas/droptriggers.sql \ + db/schemas/createtriggers.sql \ + db/schemas/comment_on.sql +DROP_SUB_DB_TARGETS := db/schemas/comment_off.sql \ + db/schemas/droptriggers.sql \ db/schemas/dropviews.sql \ db/schemas/dropindexes.sql \ db/schemas/dropfunctions.sql \ @@ -193,6 +195,15 @@ installviews dropviews \ installindexes dropindexes: $(PSQL_DEPENDS) $(MAKE) -C db/schemas $@ +.PHONY: installcomments dropcomments +## installcomments Create the comments associated with all db objects +## dropcomments Remove all comments associated with db objects +# The dependences on the sql files ensures this makefile builds the RST source. +installcomments dropcomments: $(PSQL_DEPENDS) \ + db/schemas/comment_on.sql \ + db/schemas/comment_off.sql + $(MAKE) -C db/schemas $@ + ## installtypes Install all the types into the db .PHONY: installtypes installtypes: $(PSQL_DEPENDS) createtypes.sql @@ -268,6 +279,15 @@ reindex: $(PSQL_DEPENDS) \ db/schemas/createindexes.sql ; ) \ | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y +## recomment Re-install all the comments +.PHONY: recomment +recomment: $(PSQL_DEPENDS) \ + db/schemas/comment_off.sql db/schemas/comment_on.sql + ( $(PSQL_SETUP) \ + cat db/schemas/comment_off.sql \ + db/schemas/comment_on.sql ; ) \ + | psql $(PSQL_ARGS) $(PSQL_SINGLE_TRANS) --set=ON_ERROR_STOP=y + ### retype Re-install all the data types ### (Less than useful since tables will be dropped.) #.PHONY: retype @@ -399,6 +419,24 @@ db/schemas/createviews.sql db/schemas/dropviews.sql \ db/schemas/createindexes.sql db/schemas/dropindexes.sql: $(PSQL_DEPENDS) $(MAKE) -C db/schemas $(notdir $@) +## db/schemas/comment_on.sql +## Build a file of sql that puts comments on all objects +## db/schemas/comment_off.sql +## Build a file of sql that removes comments from all +## objects +# Phony targets because we rely on sub-make for dependencies. +# It would be nice to depend on db/schemas/createtables.sql etc., +# because those will get updated timestamps if any objects change +# that would be a way to check for new or changed objects. +# But that saves no work for 2 reasons, the sub-make calls a sub-make +# and always runs, relying on the sub-make for dependencies. +# And schemas/Makefile makes the files as a dependency; this would +# have to be changed to a rule that calls make on itself. +.PHONY: db/schemas/comment_on.sql db/schemas/comment_off.sql +db/schemas/comment_on.sql \ +db/schemas/comment_off.sql: $(HTML_RST_FILES) $(PSQL_DEPENDS) install_schemas + $(MAKE) -C db/schemas $(notdir $@) + ## db/createtypes.sql Build file of sql which makes data types ## db/droptypes.sql Build file of sql which drops data types ## db/creategroups.sql @@ -407,6 +445,12 @@ db/schemas/createindexes.sql db/schemas/dropindexes.sql: $(PSQL_DEPENDS) $(DB_TARGETS) db/creategroups.sql: %.sql: %.m4 $(DB_DEPENDS) m4 $(M4_DB_INCLUDE_ARGS) $< > $@ +## db/schemas/gen_func_comment_tmpl.sql +## Build a file of sql to make a helper function +## so the build process can find functions to comment +db/schemas/gen_func_comment_tmpl.sql: + $(MAKE) -C db/schemas gen_func_comment_tmpl.sql + # It'd be nice to be able to empty the db of all data, but there's # no point until there's some regression testing. #.PHONY: empty diff --git a/make_files/make_docs.mk b/make_files/make_docs.mk index 46e1c92..d906c70 100644 --- a/make_files/make_docs.mk +++ b/make_files/make_docs.mk @@ -56,6 +56,9 @@ HTML_TARGETDIR:= $(HTML_TARGET) endif BUILDERNAME := dirhtml endif +# Exported because db/schemas/Makefile needs it to find the comments +# in the docs. +export HTML_SRCDIR # # Variables @@ -139,7 +142,7 @@ SOURCE_BASE := $(filter-out %$(EMACS_BK_CHR) $(M4_FILES), \ OLD_HTML_FILES := $(call localize,$(call rwildcard,$(HTML_SRCDIR)/,*)) OLD_LATEX_FILES := $(call localize,$(call rwildcard,$(LATEX_SRCDIR)/,*)) -# The html files to build and the directories that contain the files +# The html files to build and the directories that contain the files. HTML_RST_FILES := \ $(patsubst %.m4,%.rst,\ $(subst /$(STOCK_SRCDIR)/,/$(HTML_SRCDIR)/,$(M4_FILES))) -- 2.34.1