From 18bc91d22810489610341e80bca8637f20e048cc Mon Sep 17 00:00:00 2001 From: "Karl O. Pinc" Date: Tue, 5 Jan 2021 15:44:56 -0600 Subject: [PATCH] Initial commit --- .coveragerc | 8 + .gitignore | 11 + LICENSE.txt | 661 ++++++++++++++++++ MANIFEST.in | 9 + Makefile | 25 + Makefile_pgwui.mk | 253 +++++++ README.rst | 124 ++++ help.mk | 21 + setup.cfg | 3 + setup.py | 181 +++++ src/pgwui_bulk_upload/VERSION | 1 + src/pgwui_bulk_upload/__init__.py | 22 + src/pgwui_bulk_upload/check_settings.py | 49 ++ src/pgwui_bulk_upload/exceptions.py | 191 +++++ src/pgwui_bulk_upload/pgwui_bulk_upload.py | 50 ++ .../templates/bulk_upload.mak | 100 +++ src/pgwui_bulk_upload/views/__init__.py | 0 src/pgwui_bulk_upload/views/bulk_upload.py | 643 +++++++++++++++++ tests/test_check_settings.py | 70 ++ tests/test_pgwui_bulk_upload.py | 104 +++ tests/views/test_bulk_upload.py | 243 +++++++ tox.ini | 37 + 22 files changed, 2806 insertions(+) create mode 100644 .coveragerc create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 Makefile_pgwui.mk create mode 100644 README.rst create mode 100644 help.mk create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/pgwui_bulk_upload/VERSION create mode 100644 src/pgwui_bulk_upload/__init__.py create mode 100644 src/pgwui_bulk_upload/check_settings.py create mode 100644 src/pgwui_bulk_upload/exceptions.py create mode 100644 src/pgwui_bulk_upload/pgwui_bulk_upload.py create mode 100644 src/pgwui_bulk_upload/templates/bulk_upload.mak create mode 100644 src/pgwui_bulk_upload/views/__init__.py create mode 100644 src/pgwui_bulk_upload/views/bulk_upload.py create mode 100644 tests/test_check_settings.py create mode 100644 tests/test_pgwui_bulk_upload.py create mode 100644 tests/views/test_bulk_upload.py create mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..34f3bd7 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +branch = True +source = + pgwui_bulk_upload + views + +[report] +show_missing = True diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a40e233 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__/ +.pytest_cache/ +.cache/ +.coverage +.tox/ +build/ +devel/ +dist/ +docs/build/ +README.html +src/pgwui_bulk_upload.egg-info/ diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..dba13ed --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..703be18 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +recursive-include tests *.py +# Include Makefile includes +include *.mk +include .coveragerc +include LICENSE.txt +include Makefile +include src/pgwui_bulk_upload/VERSION +include src/pgwui_bulk_upload/templates/*.mak +include tox.ini diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fc50561 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +# Copyright (C) 2016, 2017, 2018, 2019 The Meme Factory, Inc. +# http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc kop@karlpinc.com + +PGWUI_COMPONENT := pgwui_bulk_upload + +include Makefile_pgwui.mk diff --git a/Makefile_pgwui.mk b/Makefile_pgwui.mk new file mode 100644 index 0000000..3b2607d --- /dev/null +++ b/Makefile_pgwui.mk @@ -0,0 +1,253 @@ +# Copyright (C) 2016, 2017, 2018, 2019, 2020 The Meme Factory, Inc. +# http://www.karlpinc.com/ + +# This file is part of PGWUI. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +# +# Common Makefile for PGWUI projects +# + +# Variables to customize: +# PGWUI_COMPONENT := pgwui_example + + +# This works on debian... +PYTHON_EXE=python3 +VIRTUALENV=virtualenv -p ${PYTHON_EXE} +# (Optional) where pyenv is installed (git clone somepath) +# For running tox against multiple python versions +PYENV_INSTALLATION := $(HOME)/.pyenv +PYENV_BIN := $(PYENV_INSTALLATION)/bin + +TARGETS = +EXTRA_TARGETS = README.html +SETUPTOOLS_STUFF = build dist src/${PGWUI_COMPONENT}.egg-info +PYTEST_STUFF = .cache \ + src/${PGWUI_COMPONENT}/__pycache__ \ + src/${PGWUI_COMPONENT}/views/__pycache__ \ + tests/__pycache__ tests/views/__pycache__ +COVERAGE_STUFF = .coverage +TOX_STUFF = .tox + +include help.mk + +all: ${TARGETS} + +## +## Usage: make TARGET +## +## TARGET is one of: +# + +## README.html Make html version of README.rst +README.html: README.rst + rst2html --strict README.rst > README.html + +## publish Update all public-facing info +## ("check-manifest" + "upload" + "push") +.PHONY: publish +publish: check-manifest upload push + +## run_tests Run regression tests +.PHONY: run_tests +run_tests: devel/testenv + if [ -x $(PYENV_BIN)/pyenv ] ; then \ + (set -e ; \ + export PYENV_ROOT=$(PYENV_INSTALLATION) ; \ + export PATH="$(PYENV_BIN):$$PATH" ; \ + bash -c 'set -e ; \ + eval "$$(pyenv init -)" ; \ + pyenv shell system $$(pyenv versions --bare) ; \ + devel/testenv/bin/tox --skip-missing-interpreters' ) ; \ + else \ + devel/testenv/bin/tox --skip-missing-interpreters ; \ + fi + +## dist Create sdist Python package in ./dist +.PHONY: dist +dist: + rm -rf dist + ${PYTHON_EXE} setup.py sdist + + +# Expected targets + +## clean Removed generated data +.PHONY: clean +clean: + #$(MAKE) -C docs clean + rm -rf ${TARGETS} ${EXTRA_TARGETS} ${SETUPTOOLS_STUFF} \ + ${PYTEST_STUFF} ${COVERAGE_STUFF} ${TOX_STUFF} devel + +## +## Less used TARGETS are: +# + + +# Useless targets + +## register_test Register project at pypi test site +.PHONY: register_test +register_test: + ${PYTHON_EXE} setup.py register -r https://testpypi.python.org/pypi + +## register Register project at pypi live site +.PHONY: register +register: + ${PYTHON_EXE} setup.py register + + +# Not so useless targets, used by the package owners for package management + +## upload_test Upload project to pypi test site +.PHONY: upload_test +upload_test: + ${PYTHON_EXE} setup.py sdist upload -r https://testpypi.python.org/pypi + +## upload Upload project to pypi live site +.PHONY: upload +upload: + ${PYTHON_EXE} setup.py sdist upload + +## push Push repo changes to public repo +## (Hooks at bitbucket update readthedocs) +.PHONY: push + git push + +## check Quick run of all regression tests, +## only with the default python +.PHONY: check +check: devel/pytest + devel/pytest/bin/flake8 . + devel/pytest/bin/py.test --cov=${PGWUI_COMPONENT} tests + +## check-unit Quick run of unit regression tests, +## only with the default python +.PHONY: check-unit +check-unit: devel/pytest + devel/pytest/bin/flake8 . + devel/pytest/bin/py.test -m unittest --cov=${PGWUI_COMPONENT} tests + +## check-integration Quick run of integration regression tests, +## only with the default python +.PHONY: check-integration +check-integration: devel/pytest + devel/pytest/bin/flake8 . + devel/pytest/bin/py.test -m integrationtest \ + --cov=${PGWUI_COMPONENT} tests + +## check-debug Start a regression test and drop into the Pdb +## debugger. Variables designate the test file +## and test function, i.e.: +## FILE=test_mymodule.py TEST=test_somefunction \ +## make check-debug +.PHONY: check-debug +check-debug: devel/pytest + devel/pytest/bin/flake8 . + devel/pytest/bin/py.test --trace tests/${FILE}::${TEST} + +## pudb Run the python pudb debugger +.PHONY: pudb +pudb: devel/pudb + # echo 'import pudb; pu.db' | devel/pudb/bin/python + devel/pudb/bin/python + +## devel/check-manifest +## Create a check-manifest venv +devel/check-manifest: devel + [ -d devel/check-manifest ] \ + || ( ${VIRTUALENV} devel/check-manifest ; \ + devel/check-manifest/bin/pip install --upgrade pip ; \ + devel/check-manifest/bin/pip install --upgrade setuptools ; \ + devel/check-manifest/bin/pip install check-manifest ) + +## check-manifest Run check-manifest +check-manifest: devel/check-manifest + devel/check-manifest/bin/check-manifest + +## update_check-manifest +## Upgrade check-manifest venv to latest +## versions +update_check-manifest: devel/check-manifest + devel/check-manifest/bin/pip -U + +## update_testenv Upgrade test venv to latest versions +update_testenv: devel/testenv + devel/testenv/bin/pip -U + +## help Show this help +## +## This Makefile does not work with the docs. See: +## cd docs; make help + + +# Development related targets + +# Re-create development environment when setup.py changes +devel: setup.py + rm -rf devel ${TOX_STUFF} + mkdir devel + +# virtualenv for development +devel/testenv: devel + [ -d devel/testenv ] \ + || ( ${VIRTUALENV} devel/testenv ; \ + devel/testenv/bin/pip install --upgrade pip ; \ + devel/testenv/bin/pip install --upgrade setuptools ; \ + devel/testenv/bin/pip install tox ; \ + devel/testenv/bin/pip install -e '.[testing]' ; \ + ) + +# virtualenv for pytest +devel/pytest: devel dist + if [ ! -d devel/pytest ] ; then \ + ( ${VIRTUALENV} devel/pytest ; \ + devel/pytest/bin/pip install --upgrade pip ; \ + devel/pytest/bin/pip install --upgrade setuptools ; \ + devel/pytest/bin/pip install --upgrade wheel ; \ + devel/pytest/bin/pip install dist/${PGWUI_COMPONENT}-*.tar.gz ; \ + devel/pytest/bin/pip install flake8 ; \ + devel/pytest/bin/pip install pytest-cov ; \ + devel/pytest/bin/pip install -e '.[testing]' ; \ + ) \ + else \ + ( devel/pytest/bin/pip uninstall -y ${PGWUI_COMPONENT} ; \ + devel/pytest/bin/pip install dist/${PGWUI_COMPONENT}-*.tar.gz ; \ + devel/pytest/bin/pip install -e '.[testing]' ; \ + ) ; \ + fi + +# virtualenv for pudb +devel/pudb: devel dist + if [ ! -d devel/pudb ] ; then \ + ( ${VIRTUALENV} devel/pudb ; \ + devel/pudb/bin/pip install --upgrade pip ; \ + devel/pudb/bin/pip install --upgrade setuptools ; \ + devel/pudb/bin/pip install --upgrade wheel ; \ + devel/pudb/bin/pip install dist/${PGWUI_COMPONENT}-*.tar.gz ; \ + devel/pudb/bin/pip install -e '.[testing]' ; \ + devel/pudb/bin/pip install pudb ; \ + ) \ + else \ + ( devel/pudb/bin/pip uninstall -y ${PGWUI_COMPONENT} ; \ + devel/pudb/bin/pip install -e '.[testing]' ; \ + devel/pudb/bin/pip install dist/${PGWUI_COMPONENT}-*.tar.gz \ + ) ; \ + fi diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..4f13ddf --- /dev/null +++ b/README.rst @@ -0,0 +1,124 @@ +.. Copyright (C) 2018 The Meme Factory, Inc. http://www.karlpinc.com/ + + + This file is part of PGWUI_Bulk_Upload. + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Affero General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public + License along with this program. If not, see + . + + Karl O. Pinc + +.. #End Of Copyright Marker# + +PGWUI_Bulk_Upload +================= + +Short Documentation +------------------- + +PGWUI_Bulk_Upload stands for `PostgreSQL`_ Web User Interface Bulk Upload. + + +Usage +----- + +Uploads from an archive file, presently only a zip file, into +(potentially) mulltiple PostgreSQL tables. The structure of the +zip file is as follows: + +The zip file must contain one or more directories. Each directory may +contain only files. Each directory must contain a file, the map file, +which maps the directory's files to PostgreSQL tables (or views), one +file per table (or view). + +The map file is in YAML syntax, which for purposes of this document +can be thought of as a superset of JSON. It must contain a top-level +tag, `map_list`, which itself contains a list of maps, each of which +maps a file to a table or view. The table (or view) names may, +optionally, be schema qualified. An example map file might be:: + + # This file is contents.yml + map_list: + # Load the foo.csv file into the foo_table table of the default schema. + - file_map: + file: foo.csv + relation: foo_table + # Load the bar.csv file into the bar_view uploadable-view of the default + # schema. + - file_map: + file: bar.csv + relation: bar_view + # Load the baz.csv file into the baz_table table of the meta schema. + - file_map: + file: baz.csv + relation: meta.baz_table + +The files within each directory are uploaded in the order in which +they are listed in the map file. The directories in the zip file are +processed in alphabetical order. + +Top level tags, other than the `map_list` tag, are ignored. + +It is recommended to enclose file names which contain spaces, or begin +with a digit, in single quotes. + +The `yamllint` syntax and style checker can be helpful when +constructing YAML files. See: https://github.com/adrienverge/yamllint + +The default name of the map file is `contents.yml`, but this may be +changed with the `pgwui_bulk_upload:map_file` setting. + + +URL Configuration +----------------- + +The default URL for PGWUI_Bulk_Upload looks like +*https://www.example.com/upload*. + +See the PGWUI_Server documentation for how to configure a different +URL. + + +Development Status +------------------ + + +Complete Documentation +---------------------- + +The complete documentation set can be found on the PGWUI_Upload home +page at http://pgwui_bulk_upload.readthedocs.io/. + + +License +------- + +Except for files otherwise marked, distributed WITHOUT ANY WARRANTY +under the terms of the GNU Affero General Public License, version 3 or +a later version at your option. See the copyright notices at the top +of each file and the LICENSE.txt file for details. + +Acknowledgments +--------------- + +The PGWUI_Bulk_Upload code is based on the GMI_Pyramid sub-system created for +the `Gombe Mother Infant Database Project +`_. Support for extracting PGWUI_Core_ +from GMI_Pyramid, its Python packaging, and further enhancement was +provided by `The Dian Fossey Gorilla Fund +`_. + + +.. _PGWUI_Core: http://pgwui_core.readthedocs.io/ +.. _PostgreSQL: https://www.postgresql.org/ diff --git a/help.mk b/help.mk new file mode 100644 index 0000000..9893c10 --- /dev/null +++ b/help.mk @@ -0,0 +1,21 @@ +# Generate help summary from Makefile content +# Copyright (C) 2019 The Meme Factory, Inc. www.karlpinc.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Use this by putting the following line into your Makefile: +# include help.mk + +help: + @grep -Eh '^##($$| )' $(MAKEFILE_LIST) | sed -E 's/^##($$| )//' diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..24eff81 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[bdist_wheel] +# Work with both python 2 and 3. +universal=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..66088a7 --- /dev/null +++ b/setup.py @@ -0,0 +1,181 @@ +"""A setuptools based setup module. + +Copied from: +https://github.com/pypa/sampleproject +It's license applies to this file. +""" +# Copyright (c) 2016 The Python Packaging Authority (PyPA) +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +# To use a consistent encoding +from codecs import open +from os import path + + +# Get the long description from the README file +def filter_readme(): + '''Return README content with copyright at top removed''' + lines = [] + past_copyright = False + with open(path.join(here, 'README.rst'), encoding='utf-8') as f: + for line in f: + if past_copyright: + lines.append(line) + else: + if line == '.. #End Of Copyright Marker#\n': + past_copyright = True + return ''.join(lines) + + +here = path.abspath(path.dirname(__file__)) + +# Get program version +with open(path.join(here, 'src', 'pgwui_bulk_upload', 'VERSION'), + encoding='utf-8') as version_file: + version = version_file.read().strip() + +# Get the long description from the README file +long_description = filter_readme() + +# Pyramid testing requirements +tests_require = [ + 'pgwui_testing==' + version, + 'WebTest >= 1.3.1', # py3 compat + 'pytest>=3.7.4', + 'pytest-cov', +] + +setup( + name='pgwui_bulk_upload', + + # Versioning is major.minor.fixes. Major releases change (after 1.0.0) + # when backward incompatibility is introduced. Minor releases introduce + # new features. Fix releases introduce fixes. + version=version, + + description=( + 'Upload into many PostgreSQL tables, assisting data cleanup, ' + 'via the web.'), + long_description=long_description, + long_description_content_type='text/x-rst', + + # The project's main homepage. + url='http://pgwui_upload.readthedocs.io/', + + # Author details + author='Karl O. Pinc', + author_email='kop@karlpinc.com', + + # Choose your license + license='AGPLv3+', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + # Indicate who your project is intended for + 'Environment :: Web Environment', + 'Framework :: Pyramid', + 'Intended Audience :: End Users/Desktop', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + 'Intended Audience :: System Administrators', + 'Topic :: Database :: Front-Ends', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Scientific/Engineering', + 'Topic :: Utilities', + + # Pick your license as you wish (should match "license" above) + ('License :: OSI Approved ' + ':: GNU Affero General Public License v3 or later (AGPLv3+)'), + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + + # What does your project relate to? + keywords=[ + 'database front end', + 'interface', + 'Postgres', + 'Postgres user interface', + 'Postgres web interface', + 'PostgreSQL', + 'Postgres front end', + 'user interface', + 'web', + 'web front end', + ], + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + packages=find_packages('src'), + package_dir={'': 'src'}, + + # Run-time dependencies. + install_requires=[ + 'attrs', + 'markupsafe', + 'pgwui_common==' + version, + 'pgwui_upload_core==' + version, + 'pyramid', + 'pyyaml', + ], + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[testing] + extras_require={ + 'testing': tests_require, + }, + + # If there are data files included in your packages that need to be + # installed, specify them here. If using Python 2.6 or less, then these + # have to be included in MANIFEST.in as well. + package_data={ + 'pgwui_bulk_upload': [ + 'templates/*.mak', + 'VERSION', + ], + }, + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. + # + # Setup an entry point to support PGWUI autoconfigure discovery. + entry_points={ + 'pgwui.components': '.pgwui_bulk_upload = pgwui_bulk_upload', + 'pgwui.check_settings': + '.pgwui_bulk_upload = pgwui_bulk_upload.check_settings:check_settings'} +) diff --git a/src/pgwui_bulk_upload/VERSION b/src/pgwui_bulk_upload/VERSION new file mode 100644 index 0000000..a918a2a --- /dev/null +++ b/src/pgwui_bulk_upload/VERSION @@ -0,0 +1 @@ +0.6.0 diff --git a/src/pgwui_bulk_upload/__init__.py b/src/pgwui_bulk_upload/__init__.py new file mode 100644 index 0000000..450465c --- /dev/null +++ b/src/pgwui_bulk_upload/__init__.py @@ -0,0 +1,22 @@ +# Copyright (C) 2018, 2020 The Meme Factory, Inc. http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +from .pgwui_bulk_upload import includeme # noqa: F401 diff --git a/src/pgwui_bulk_upload/check_settings.py b/src/pgwui_bulk_upload/check_settings.py new file mode 100644 index 0000000..cefcac6 --- /dev/null +++ b/src/pgwui_bulk_upload/check_settings.py @@ -0,0 +1,49 @@ +# Copyright (C) 2020 The Meme Factory, Inc. http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +import pgwui_upload_core + + +PGWUI_COMPONENT = 'pgwui_bulk_upload' + + +def check_settings(component_config): + '''Check that all pgwui_upload specific settings are good. + This includes: + checking for unknown settings + checking for missing required settings + checking the boolean settings + checking that the values of other settings are valid + ''' + errors = [] + + all_setngs = ['map_file'] + all_setngs.extend(pgwui_upload_core.check_settings.UPLOAD_SETTINGS) + + required_setngs = ['map_file'] + required_setngs.extend(pgwui_upload_core.check_settings.REQUIRED_SETTINGS) + + errors.extend(pgwui_upload_core.check_settings.check_settings( + PGWUI_COMPONENT, all_setngs, required_setngs, + pgwui_upload_core.check_settings.BOOLEAN_SETTINGS, + component_config)) + + return errors diff --git a/src/pgwui_bulk_upload/exceptions.py b/src/pgwui_bulk_upload/exceptions.py new file mode 100644 index 0000000..3d953bb --- /dev/null +++ b/src/pgwui_bulk_upload/exceptions.py @@ -0,0 +1,191 @@ +# Copyright (C) 2020 The Meme Factory, Inc. http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +from pgwui_core import exceptions as core_ex + + +# PGWUI setting related exceptions + +class Error(core_ex.Error): + pass + + +# Exceptions that require detail + +class NotAZipfileError(Error): + def __init__(self, filename): + super().__init__( + 'No zip file uploaded', + f'The uploaded file ({filename}) is not recognized as a zip file') + + +class CannotUnarchiveError(Error): + def __init__(self, filename, err): + super().__init__( + 'Cannot unzip', + (f'The uploaded file ({filename}) cannot be unzipped, ' + 'the error reported is: err')) + + +class NotADirectoryError(Error): + def __init__(self, filename, name): + super().__init__( + 'Not a directory', + f'The uploaded file ({filename}) has a top-level ' + f'entry ({name}) that is not a directory', + 'The entries at the top of the zip file must be directories') + + +class EmptyArchiveError(Error): + def __init__(self, filename, err): + super().__init__( + 'Empty zip file', + f'The uploaded file ({filename}) has no content') + + +class NotAFileError(Error): + def __init__(self, filename, name): + super().__init__( + 'Not a file', + f'The uploaded file ({filename}) has a 2nd-level ' + f'entry ({name}) that is not a file', + 'The directories in the zip file may only contain files') + + +class NoMapfileError(Error): + def __init__(self, filename, map_file): + super().__init__( + 'Missing map file', + f'The map file ({map_file}) not found in the ' + f'uploaded file ({filename})', + 'The directories in the zip file must contain a file which ' + 'maps file names to table names') + + +class CannotReadError(Error): + def __init__(self, filename, exp): + super().__init__( + f'Cannot open ({filename}) for reading', + f'The error is: {exp}') + + +class BadMapFileError(Error): + pass + + +class BadMapfileYAMLError(BadMapFileError): + def __init__(self, filename, map_file, exp): + super().__init__( + 'Bad YAML in map file', + f'The map file ({map_file}) in the ' + f'uploaded file ({filename}) cannot be parsed as YAML: {exp}', + 'The directories in the zip file must contain a map file in YAML ' + 'format to maps file names to table names') + + +class NoMapListError(BadMapFileError): + def __init__(self, filename): + super().__init__( + f'Missing "map_list" tag in map file ({filename})', + 'The map file does not have "map_list" as a top-level tag') + + +class BadMapListError(BadMapFileError): + def __init__(self, filename): + super().__init__( + f'The "map_list" tag in map file ({filename}) is not a list', + 'The map file does not have a list in the "map_list" tag') + + +class BadMapListEntryError(BadMapFileError): + def __init__(self, filename, item): + super().__init__( + f'The "map_list" tag of map file ({filename}) contains an entry ' + 'which is not a map', + f'The list item ({item}) is not a map') + + +class MustBeStringError(BadMapFileError): + def __init__(self, filename, count, key, value): + super().__init__( + f'The "map_list" tag of map file ({filename}) contains a map ' + 'with an invalid value', + f'The value ({value}) of key ({key}) of map_list item number ' + f'{count} is not a string') + + +class MissingMapListTagError(BadMapFileError): + def __init__(self, filename, count, key): + super().__init__( + f'The "map_list" tag of map file ({filename}) contains a map ' + 'missing a required tag', + f'The tag ({key}) is missing from map_list item number {count}') + + +class ExtraMapListTagError(BadMapFileError): + def __init__(self, filename, count, key): + super().__init__( + f'The "map_list" tag of map file ({filename}) contains a map ' + 'with an unknown tag', + f'The tag ({key}) is not recognized in map_list item ' + f'number {count}') + + +class MissingFileMapTagError(BadMapFileError): + def __init__(self, filename, count, key): + super().__init__( + f'The "map_list" tag of map file ({filename}) contains a ' + 'file_map map missing a required tag', + f'The tag ({key}) is missing from map_list item number {count}') + + +class ExtraFileMapTagError(BadMapFileError): + def __init__(self, filename, count, key): + super().__init__( + f'The "map_list" tag of map file ({filename}) contains a ' + 'file_map map with an unknown tag', + f'The tag ({key}) is not recognized in map_list item ' + f'number {count}') + + +class DuplicateMappingError(BadMapFileError): + def __init__(self, filename, pos0, pos1, file, relation): + super().__init__( + f'The "map_list" tag of map file ({filename}) specifies duplicate ' + 'uploads', + f'The list entries {pos0} and {pos1} both upload file ({file}) ' + f'into relation ({relation})') + + +class MissingFileError(BadMapFileError): + def __init__(self, filename, item): + super().__init__( + f'The "map_list" tag of map file ({filename}) references a ' + "file not in the zip file's directory", + f'Missing file ({item})') + + +class ExtraFileError(BadMapFileError): + def __init__(self, filename, extrafile): + super().__init__( + f'The "map_list" tag of the map file ({filename}) does not ' + 'contain an entry for a file in the zip file', + f'Extra file ({extrafile})') diff --git a/src/pgwui_bulk_upload/pgwui_bulk_upload.py b/src/pgwui_bulk_upload/pgwui_bulk_upload.py new file mode 100644 index 0000000..9600e2a --- /dev/null +++ b/src/pgwui_bulk_upload/pgwui_bulk_upload.py @@ -0,0 +1,50 @@ +# Copyright (C) 2018, 2020 The Meme Factory, Inc. http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +'''Provide a way to configure this PGWUI component. +''' +PGWUI_COMPONENT = 'pgwui_bulk_upload' +DEFAULT_BULK_UPLOAD_ROUTE = '/bulk_upload' +DEFAULT_BULK_UPLOAD_MENU_LABEL = ( + 'bulk_upload -- Upload Many Files Into PostgreSQL') +DEFAULT_MAP_FILE = 'contents.yml' + + +def establish_default_settings(config): + '''Setup all the defaults: + menu label, map file + ''' + settings = config.get_settings() + pgwui = settings.setdefault('pgwui', dict()) + pgwui.setdefault(PGWUI_COMPONENT, dict()) + bulk_upload_settings = pgwui[PGWUI_COMPONENT] + bulk_upload_settings.setdefault( + 'menu_label', DEFAULT_BULK_UPLOAD_MENU_LABEL) + bulk_upload_settings.setdefault( + 'map_file', DEFAULT_MAP_FILE) + + +def includeme(config): + '''Pyramid configuration for PGWUI_Bulk_Upload + ''' + establish_default_settings(config) + config.add_route(PGWUI_COMPONENT, DEFAULT_BULK_UPLOAD_ROUTE) + config.scan() diff --git a/src/pgwui_bulk_upload/templates/bulk_upload.mak b/src/pgwui_bulk_upload/templates/bulk_upload.mak new file mode 100644 index 0000000..800975b --- /dev/null +++ b/src/pgwui_bulk_upload/templates/bulk_upload.mak @@ -0,0 +1,100 @@ +<%doc> + Copyright (C) 2015, 2018, 2020 The Meme Factory, Inc. + http://www.karlpinc.com/ + + This file is part of PGWUI_Bulk_Upload. + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Affero General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public + License along with this program. If not, see + . + + Template for generic bulk_upload page. + + Karl O. Pinc + + This template uses the following variables in it's context: + + ... + + + +<%! + from pgwui_common.path import asset_abspath + import pgwui_core.exceptions as core_ex + + upload_base_mak = asset_abspath('pgwui_upload_core:templates/upload.mak') +%> + +<%inherit file="${upload_base_mak}" /> + +<%block name="title">${pgwui['pgwui_bulk_upload']['menu_label']} +<%block name="meta_keywords"> + + + +<%block name="meta_description"> + + + +<%block name="action_success"> +

Successfull bulk upload of ${filename}; + inserted ${inserted_rows} rows from ${len(stats)} + files! +

+ + % if len(stats) != len(relation_stats): + <%doc> There were multiple files loaded into the same relation + % for (relation, count) in relation_stats: +

Inserted ${count} rows into relation (${relation})

+ % endfor + % endif + % for (path, lines, relation) in stats: +

Uploaded the ${lines} lines of file (${path}) into (${relation})

+ % endfor + + +<%def name="file_errors(errors, filepath)"> + <% + if filepath is not None: + file_info = statmap[error.filepath] + file_lines = file_info[0] + relation = file_info[1] + self.error_file_summary(len(errors), file_lines, filepath, relation) + self.error_items(errors) + %> + + +<%block name="error_report"> + % if e_cnt: +

${e_cnt} errors found.

+ <% + last_filepath = None + f_errors = [] + for error in errors: + if isinstance(error, core_ex.UploadError): + if error.filepath != last_filepath: + file_errors(f_errors, last_filepath) + last_filepath = error.filepath + f_errors = [] + f_errors.append(error) + self.file_errors(f_errors, last_filepath) + %> + % endif + + +

Upload Into Multiple Tables

+ +${self.upload_form([])} diff --git a/src/pgwui_bulk_upload/views/__init__.py b/src/pgwui_bulk_upload/views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pgwui_bulk_upload/views/bulk_upload.py b/src/pgwui_bulk_upload/views/bulk_upload.py new file mode 100644 index 0000000..f753251 --- /dev/null +++ b/src/pgwui_bulk_upload/views/bulk_upload.py @@ -0,0 +1,643 @@ +# Copyright (C) 2015, 2018, 2020 The Meme Factory, Inc. +# http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +from pyramid.view import view_config +import attr +import logging +import os +import os.path +import pathlib +import psycopg2 +import tempfile +import zipfile + +import yaml + +from pgwui_common.view import auth_base_view +import pgwui_core.exceptions as core_ex + +from pgwui_core.core import ( + UploadNullFileWTForm, + UploadEngine, + DataLineProcessor, + UploadNullMixin, + UploadDoubleFileForm, + UploadData, + escape_eol, + is_checked, +) +from pgwui_upload_core.views.upload import ( + BaseTableUploadHandler, +) + +import pgwui_bulk_upload.exceptions as ex + + +log = logging.getLogger(__name__) + + +def map_description(filepath, relation): + return f'Error uploading ({filepath}) into ({relation})' + + +def archive_path(path): + '''Return, as text, the path of a file within the archive + ''' + return os.path.join(*[os.fsdecode(part) for part in path.parts[-2:]]) + + +class BulkUploadForm(UploadNullMixin, UploadDoubleFileForm): + ''' + Acts like a dict, but with extra methods. + + Attributes: + uh The UploadHandler instance using the form + + Methods: + read() Load form from pyramid request object. + ''' + def read(self): + ''' + Read form data from the client + ''' + # Read parent's data + super().read() + + def write(self, result, errors): + ''' + Produces the dict pyramid will use to render the form. + ''' + response = super().write(result, errors) + return super().write_response(response) + + +class SaveLine(DataLineProcessor): + def __init__(self, ue, uh, insert_map): + ''' + ue UploadEngine instance + uh UploadHandler instance + insert_map Dict mapping file to insert statement used to insert + into db. (psycopg2 formatted for substituion) + ''' + super().__init__(ue, uh) + self.insert_map = insert_map + + def eat(self, bulk_data): + ''' + Upload a line of data into the db. + + bulk_data A UploadBulkData value. A tuple: + (filepath, relation, + thunk returning UploadDataLine instance) + ''' + try: + udl = bulk_data[2]() + self.cur.execute( + self.insert_map[bulk_data[0]], udl.tuples) + except (core_ex.DataLineError, core_ex.DBError) as exp: + exp.color(map_description(bulk_data[0], bulk_data[1]), + bulk_data[0], bulk_data[1]) + raise exp + except psycopg2.DatabaseError as exp: + myexp = core_ex.DBDataLineError(udl, exp) + myexp.color(map_description(bulk_data[0], bulk_data[1]), + bulk_data[0], bulk_data[1]) + raise myexp + + +class UploadBulkData(UploadData): + def __init__(self, fileo, file_fmt, null_data, null_rep, + filepath, relation, trim=True): + ''' + fileo Uploaded file object + file_fmt File format: CSV or TAB + null_data (boolean) Uploaded data contains nulls + null_rep Uploaded string representation of null + filepath Path of file (zip_root relative) + relation Relation, possibly schema qualified + trim (boolean) Trim leading and trailing whitespace? + ''' + super().__init__(fileo, file_fmt, null_data, null_rep, trim=True) + self.filepath = filepath + self.relation = relation + + +@attr.s +class UploadDir(): + '''Uploaded archive directory + + It is iterable, consisting of SaveLine instances + ''' + uf = attr.ib() + dentry = attr.ib() + map_file = attr.ib() + filedata = attr.ib(factory=list) + + def get_file_map(self, yaml_file): + try: + with yaml_file.open() as yaml_fh: + return yaml.safe_load(yaml_fh) + except yaml.YAMLError as exp: + raise ex.BadMapfileError( + self.uf['filename'], archive_path(yaml_file), exp) + + def check_tag(self, errors, yaml_file, count, file_map, tag): + '''Confirm that the tag exists and holds a string; + remove the tag from the map + ''' + try: + if not isinstance(file_map[tag], str): + errors.append(ex.MustBeStringError( + archive_path(yaml_file), count, tag, file_map[tag])) + except KeyError: + errors.append(ex.MissingFileMapTagError( + archive_path(yaml_file), count, tag)) + else: + del file_map[tag] + + def extra_tags(self, errors, yaml_file, count, map_item, execp): + '''Confirm that there are no unrecognized tags + ''' + for key in map_item: + errors.append(execp(archive_path(yaml_file), count, key)) + + def validate_file_map(self, errors, yaml_file, count, file_map): + '''Confirm that a file_map contains the right tags, if not + add to errors + ''' + my_file_map = file_map.copy() + self.check_tag(errors, yaml_file, count, my_file_map, 'file') + self.check_tag(errors, yaml_file, count, my_file_map, 'relation') + self.extra_tags(errors, yaml_file, count, my_file_map, + ex.ExtraFileMapTagError) + + def validate_map_item(self, errors, yaml_file, count, map_item): + '''Confirm that a map_item is a dict with the right tags, if not + add to errors + ''' + if not isinstance(map_item, dict): + errors.append(ex.BadMapListEntryError( + archive_path(yaml_file), map_item)) + return + my_map_item = map_item.copy() + my_map_item.pop('file_map', None) + self.extra_tags(errors, yaml_file, count, my_map_item, + ex.ExtraMapListTagError) + if 'file_map' not in map_item: + errors.append(ex.MissingMapListTagError( + archive_path(yaml_file), count, 'file_map')) + return + self.validate_file_map(errors, yaml_file, count, map_item['file_map']) + + def detect_duplicates(self, errors, yaml_file, map_list): + '''Add to errors if the same file is uploaded into the same table + more than once + ''' + pairs = dict() + count = 1 + for map_item in map_list: + file_map = map_item['file_map'] + file = file_map['file'] + relation = file_map['relation'] + pair = (file, relation) + if pair in pairs: + errors.append(ex.DuplicateMappingError( + archive_path(yaml_file), pairs[pair], + count, file, relation)) + else: + pairs[pair] = count + count += 1 + + def validate_map_list(self, yaml_file, map_list): + '''Confirm that the map list contains the right YAML structure + ''' + errors = [] + count = 1 + for map_item in map_list: + self.validate_map_item(errors, yaml_file, count, map_item) + count += 1 + self.detect_duplicates(errors, yaml_file, map_list) + if errors: + raise core_ex.MultiError(errors) + + def validate_mapfile(self, yaml_file, file_map): + '''Confirm that the file map has the right YAML structure + ''' + if not isinstance(file_map, dict) or 'map_list' not in file_map: + raise ex.NoMapListError(archive_path(yaml_file)) + + map_list = file_map['map_list'] + if not isinstance(map_list, list): + raise ex.BadMapListError(archive_path(yaml_file)) + + self.validate_map_list(yaml_file, map_list) + + def collect_files(self, dir_name): + '''Return a list of all the file names, but the map file, in dir_name + ''' + map_file = self.map_file + file_names = [] + with os.scandir(dir_name) as direntry: + for file in direntry: + name = os.fsdecode(file.name) + if name != map_file: + file_names.append(name) + return file_names + + def match_map_to_fs(self, dir_name, yaml_file, file_map): + '''Produce errors if the files in the dir do not match the names + in the file map + ''' + errors = [] + file_names = set(self.collect_files(dir_name)) + item_files = set([item['file_map']['file'] + for item in file_map['map_list']]) + + extra_items = item_files - file_names + for item in extra_items: + errors.append(ex.MissingFileError( + archive_path(yaml_file), archive_path(dir_name / item))) + + extra_files = file_names - item_files + for extrafile in extra_files: + errors.append(ex.ExtraFileError( + archive_path(yaml_file), archive_path(dir_name / extrafile))) + + if errors: + raise core_ex.MultiError(errors) + + def get_filedata(self, dir_name, file_map): + '''Return a list of UploadData instances or raise an error + + The list is in the order given in the map file. + ''' + uf = self.uf + filedata = [] + errors = [] + for item in file_map['map_list']: + fmap = item['file_map'] + name = dir_name / fmap['file'] + try: + fh = name.open('rb') + except OSError as exp: + errors.append(ex.CannotReadError(archive_path(name), exp)) + else: + filedata.append(UploadBulkData(fh, + uf['upload_fmt'], + uf['upload_null'], + uf['null_rep'], + archive_path(name), + fmap['relation'], + trim=uf['trim_upload'])) + if errors: + raise core_ex.MultiError(errors) + return filedata + + def open(self, dir_name): + ''' + Initialize object, return number of files to upload + dir_name Path of directory to upload, a pathlib.Path + ''' + yaml_file = dir_name / self.map_file + file_map = self.get_file_map(yaml_file) + self.validate_mapfile(yaml_file, file_map) + self.match_map_to_fs(dir_name, yaml_file, file_map) + self.filedata = self.get_filedata(dir_name, file_map) + return len(self.filedata) + + def __iter__(self): + for data in self.filedata: + filepath = data.filepath + relation = data.relation + for thunk in data: + def colored_thunk(): + try: + return thunk() + except core_ex.DataLineError as ex: + ex.color(map_description(filepath, relation), + filepath, relation) + raise ex + yield lambda: (filepath, relation, colored_thunk) + + def fileinfo(self): + for fileinfo in self.filedata: + yield fileinfo + + def stats(self): + for fileinfo in self.filedata: + yield (fileinfo.filepath, fileinfo.lineno, fileinfo.relation) + + +@attr.s +class UploadArchive(): + '''Uploaded archive + + It is iterable, consisting of UploadDir instances + ''' + lineno = attr.ib(default=0) + updirs = attr.ib(factory=list) + + def open(self, uf, zip_root, map_file): + ''' + uf The upload form + zip_root Root of unarchived upload, a pathlib.Path + map_file Name of the file which maps tables to files + ''' + err_tups = [] + with os.scandir(zip_root) as archive: + for dentry in archive: + updir = UploadDir(uf, dentry, map_file) + try: + # Every file consumes a header line + self.lineno += updir.open(zip_root / dentry.name) + except ex.Error as exp: + err_tups.append((dentry.name, exp)) + except core_ex.MultiError as exp: + err_tups.extend([(dentry.name, err) for err in exp.errors]) + else: + self.updirs.append(updir) + if err_tups: + err_tups.sort(key=lambda tup: tup[0]) + raise core_ex.MultiError([tup[1] for tup in err_tups]) + + self.updirs.sort(key=lambda elmt: os.fsdecode(elmt.dentry.name)) + + def __iter__(self): + for updir in self.updirs: + for thunk in updir: + self.lineno += 1 + yield thunk + + def filedata(self): + for updir in self.updirs: + for info in updir.fileinfo(): + yield info + + def stats(self): + for updir in self.updirs: + for stat in updir.stats(): + yield stat + + +class BulkTableUploadHandler(BaseTableUploadHandler): + ''' + Attributes: + request A pyramid request instance + uf A GCUploadForm instance + session A pyramid session instance + ue + cur + map_file + temp_dir + ''' + def __init__(self, request): + ''' + request A pyramid request instance + ''' + super().__init__(request) + self.temp_dir = None + + def cleanup(self): + if self.temp_dir: + self.temp_dir.cleanup() + super().cleanup() + + def make_form(self): + ''' + Make the upload form needed by this handler. + ''' + return BulkUploadForm(self, fc=UploadNullFileWTForm) + + def get_data(self): + ''' + Return an UploadData instance, with flags set as desired. + ''' + self.data = UploadArchive() + + def validate_mapfile(self, updir, errors): + '''Add an error to errors if the map file does not exist + ''' + map_file_path = pathlib.Path(updir) / self.map_file + if not map_file_path.exists(): + errors.append(ex.NoMapfileError( + self.uf['filename'], archive_path(map_file_path))) + + def validate_updir(self, updir): + '''Add to the list of errors from a directory of an uploaded archive + ''' + errors = [] + with os.scandir(updir) as updir_entry: + for entry in updir_entry: + if not entry.is_file(): + errors.append(ex.NotAFileError( + self.uf['filename'], + archive_path(pathlib.Path(entry)))) + self.validate_mapfile(updir, errors) + return errors + + def validate_archive_structure(self): + '''Return list of errors of the archive content's structure, + sorted by top-level directory name + ''' + self.map_file = (self.request.registry.settings['pgwui'] + ['pgwui_bulk_upload']['map_file']) + err_tups = [] + count = 0 + with os.scandir(self.zip_root) as archive: + for entry in archive: + if entry.is_dir(): + err_tups.extend([ + (entry.name, error) for error in + self.validate_updir(entry)]) + else: + err_tups.append( + (entry.name, + ex.NotADirectoryError( + self.uf['filename'], os.fsdecode(entry.name)))) + count += 1 + if count == 0: + return [ex.EmptyArchiveError(self.uf['filename'])] + err_tups.sort(key=lambda tup: tup[0]) + return [tup[1] for tup in err_tups] + + def unarchive(self): + '''Unarchive the zip file. Return a list of errors. + ''' + fh = self.uf['localfh'] + try: + with zipfile.ZipFile(fh) as zfile: + zfile.extractall(self.zip_root) + except zipfile.BadZipFile as exp: + return [ex.CannotUnarchiveError(self.uf['filename'], exp)] + return self.validate_archive_structure() + + def validate_archive(self): + '''Validate the uploaded archive; un-archive it and save + a path to the decompressed archive. Return a list of errors. + ''' + fh = self.uf['localfh'] + if not zipfile.is_zipfile(fh): + return [ex.NotAZipfileError(self.uf['filename'])] + self.temp_dir = tempfile.TemporaryDirectory() + self.zip_root = pathlib.Path(self.temp_dir.name) + return self.unarchive() + + def val_input(self): + ''' + Validate input needed beyond that required to connect to the db. + + Returns: + A list of PGWUIError instances + ''' + errors = super().val_input() + + errors.extend(self.validate_archive()) + + return errors + + def insert_map(self): + '''Return a map (dict) of file paths to insert statements + ''' + quotecols = self.quote_columns() + column_quoter = self.get_column_quoter(quotecols) + + i_map = dict() + errors = [] + for fileinfo in self.data.filedata(): + try: + i_map[fileinfo.filepath] = self.build_insert_stmt( + fileinfo, fileinfo.relation, quotecols, column_quoter) + except core_ex.PGWUIError as exp: + exp.color( + map_description(fileinfo.filepath, fileinfo.relation), + fileinfo.filepath, fileinfo.relation) + errors.append(exp) + if errors: + raise core_ex.MultiError(errors) + + return i_map + + def factory(self, ue): + '''Make a db loader function from an UploadEngine. + + Input: + + Side Effects: + Yes, lots. + ''' + self.ue = ue + self.cur = ue.cur + + self.data.open(self.uf, self.zip_root, self.map_file) + + return SaveLine(self.ue, self, self.insert_map()) + + def stats(self): + return self.data.stats() + + +def statmap(stats): + '''Return map of file path to (lines, relation) tuple + ''' + smap = dict() + for stat in stats: + smap[stat[0]] = stat[1:] + return smap + + +def inserted_rows(stats, response): + '''Return the number of rows inserted + ''' + return response['lines'] - len(stats) + + +def relation_stats(stats, quote_columns): + '''Produce summaries per relation from the stats + ''' + relations = dict() + for (path, lines, relation) in stats: + if quote_columns: + key = relation.lower() + else: + key = relation + count = relations.get(key, 0) + relations[key] = count + lines - 1 + items = list(relations.items()) + items.sort(key=lambda x: x[0]) + return items + + +def log_success(response, stats, i_rows, r_stats): + if is_checked(response['csv_checked']): + upload_fmt = 'CSV' + else: + upload_fmt = 'TAB' + + msg = [('Successful bulk upload: ' + f"Archive file ({response['filename']}): " + f"DB {response['db']}: " + f"Inserted a total of {i_rows} rows: " + f'From {len(stats)} files: ')] + if len(stats) != len(r_stats): + # There were multiple files loaded into the same relation + for (relation, count) in r_stats: + msg.append( + f'Inserted {count} rows into relation ({relation}): ') + for (path, lines, relation) in stats: + msg.append( + f'Uploaded the {lines} lines of file ({path}) into ({relation}): ') + msg.append(f'Format {upload_fmt}: ' + f"Upload Null {is_checked(response['upload_null'])}: " + f"Null Rep ({escape_eol(response['null_rep'])}): " + f"Trim {is_checked(response['trim_upload'])}: " + f"By user {response['user']}") + + log.info(''.join(msg)) + + +def analyze_results(uh, response): + if response['db_changed']: + stats = list(uh.stats()) + response['stats'] = stats + response['statmap'] = statmap(stats) + i_rows = inserted_rows(stats, response) + response['inserted_rows'] = i_rows + r_stats = relation_stats(stats, uh.quote_columns()) + response['relation_stats'] = r_stats + + log_success(response, stats, i_rows, r_stats) + + return response + + +@view_config(route_name='pgwui_bulk_upload', + renderer='pgwui_bulk_upload:templates/bulk_upload.mak') +@auth_base_view +def bulk_upload_view(request): + + uh = BulkTableUploadHandler(request).init() + response = UploadEngine(uh).run() + + settings = request.registry.settings + response.setdefault('pgwui', dict()) + response['pgwui']['pgwui_bulk_upload'] = ( + settings['pgwui']['pgwui_bulk_upload']) + + return analyze_results(uh, response) diff --git a/tests/test_check_settings.py b/tests/test_check_settings.py new file mode 100644 index 0000000..c1e5ebb --- /dev/null +++ b/tests/test_check_settings.py @@ -0,0 +1,70 @@ +# Copyright (C) 2020 The Meme Factory, Inc. http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +import pytest + +import pgwui_upload_core.check_settings +import pgwui_bulk_upload.check_settings as check_settings + +from pgwui_testing import testing + +# Activiate the PGWUI pytest plugin +pytest_plugins = ("pgwui",) + + +# Module packaging test + +def test_check_setting_is_pgwui_check_settings( + pgwui_check_settings_entry_point): + '''Ensure that pgwui_bulk_upload has a pgwui.check_settings entry point + ''' + assert (pgwui_check_settings_entry_point( + 'pgwui_bulk_upload.check_settings') is True) + + +# Mocks + +mock_core_check_settings = testing.make_mock_fixture( + pgwui_upload_core.check_settings, 'check_settings') + + +# check_settings() + +@pytest.mark.unittest +def test_check_settings(mock_core_check_settings): + '''The setting checking functions are called once, the check_settings() + call returns all the errors from each mock, the 'map_file' setting + is a setting and is required + ''' + + expected_errors = ['some error'] + + mock_core_check_settings.return_value = expected_errors + + result = check_settings.check_settings({}) + + mock_core_check_settings.assert_called_once + + call_args = mock_core_check_settings.call_args[0] + assert 'map_file' in call_args[1] # 'map_file' is a setting + assert 'map_file' in call_args[2] # 'map_file' is required + + assert result == expected_errors diff --git a/tests/test_pgwui_bulk_upload.py b/tests/test_pgwui_bulk_upload.py new file mode 100644 index 0000000..dd4baaa --- /dev/null +++ b/tests/test_pgwui_bulk_upload.py @@ -0,0 +1,104 @@ +# Copyright (C) 2019, 2020 The Meme Factory, Inc. http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +import pytest +import pyramid.testing + +import pgwui_bulk_upload.pgwui_bulk_upload as pgwui_bulk_upload + +from pgwui_testing import testing + +# Activiate the PGWUI pytest plugin +pytest_plugins = ("pgwui",) + + +# Module packaging test + +def test_pgwui_bulk_upload_is_pgwui_component(pgwui_component_entry_point): + '''Ensure that pgwui_bulk_upload is a pgwui.component entry point + ''' + assert pgwui_component_entry_point('pgwui_bulk_upload') is True + + +# establish_default_settings() + +@pytest.mark.unittest +def test_establish_default_settings_default(): + '''The settings get the module's default value when no settings exist + ''' + with pyramid.testing.testConfig() as config: + + pgwui_bulk_upload.establish_default_settings(config) + + new_settings = config.get_settings() + bulk_upload_settings = new_settings['pgwui']['pgwui_bulk_upload'] + assert (bulk_upload_settings['menu_label'] + == pgwui_bulk_upload.DEFAULT_BULK_UPLOAD_MENU_LABEL) + assert (bulk_upload_settings['map_file'] + == pgwui_bulk_upload.DEFAULT_MAP_FILE) + + +@pytest.mark.unittest +def test_establish_default_settings_no_default(): + '''The settings keep their value when they exist + ''' + test_menu_label = 'test label' + test_map_file = 'somefile.yml' + + with pyramid.testing.testConfig() as config: + sample_settings = config.get_settings() + + sample_settings['pgwui'] = dict() + bulk_upload_settings = dict() + sample_settings['pgwui']['pgwui_bulk_upload'] = bulk_upload_settings + bulk_upload_settings['menu_label'] = test_menu_label + bulk_upload_settings['map_file'] = test_map_file + + pgwui_bulk_upload.establish_default_settings(config) + + new_settings = config.get_settings()['pgwui']['pgwui_bulk_upload'] + assert new_settings['menu_label'] == test_menu_label + assert new_settings['map_file'] == test_map_file + + +mock_establish_default_settings = testing.make_mock_fixture( + pgwui_bulk_upload, 'establish_default_settings') + + +# includeme() + +mock_add_route = testing.instance_method_mock_fixture('add_route') +mock_scan = testing.instance_method_mock_fixture('scan') + + +@pytest.mark.unittest +def test_includeme(mock_establish_default_settings, mock_add_route, mock_scan): + '''establish_default_settings, add_route, and scan are all called + ''' + with pyramid.testing.testConfig() as config: + mocked_add_route = mock_add_route(config) + mocked_scan = mock_scan(config) + + pgwui_bulk_upload.includeme(config) + + mock_establish_default_settings.assert_called_once() + mocked_add_route.assert_called_once() + mocked_scan.assert_called_once() diff --git a/tests/views/test_bulk_upload.py b/tests/views/test_bulk_upload.py new file mode 100644 index 0000000..4dc063a --- /dev/null +++ b/tests/views/test_bulk_upload.py @@ -0,0 +1,243 @@ +# Copyright (C) 2018, 2019, 2020 The Meme Factory, Inc. +# http://www.karlpinc.com/ + +# This file is part of PGWUI_Bulk_Upload. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with this program. If not, see +# . +# + +# Karl O. Pinc + +import pytest +from pgwui_testing import testing + +import copy +import logging +from pyramid.threadlocal import get_current_request +from pgwui_common.__init__ import includeme as pgwui_common_includeme +from pgwui_core import constants +from pgwui_bulk_upload.__init__ import includeme as pgwui_bulk_upload_includeme +from pgwui_bulk_upload.views import bulk_upload + +# Activiate our pytest plugin +pytest_plugins = ("pgwui",) + + +# Constants +CHANGED_RESPONSE = { + 'db': 'somedb', + 'db_changed': True, + 'filename': 'file', + 'lines': 5, + 'null_rep': 'NULL', + 'table': 'sometable', + 'trim_upload': constants.CHECKED, + 'upload_null': constants.CHECKED, + 'user': 'someuser', +} + +UNCHANGED_RESPONSE = {'db_changed': False} + +HOME_PAGE_SETTINGS = {'type': 'URL', + 'source': '/'} + +DEFAULT_URLS = {'pgwui_upload': '/upload', + 'pgwui_logout': '/logout', + 'home_page': '/'} + +mock_stats = testing.instance_method_mock_fixture('stats') +mock_quote_columns = testing.instance_method_mock_fixture('quote_columns') + + +# Helper classes + +class MockUploadEngine(): + def __init__(self, run_result): + self.run_result = run_result + + def run(self): + return self.run_result + + +class MockBulkTableUploadHandler(): + def __init__(self, request): + pass + + def init(self, *args): + pass + + def stats(self): + return [] + + def quote_columns(self): + pass + + +# statmap() + + +mock_statmap = testing.make_mock_fixture( + bulk_upload, 'statmap') + + +# inserted_rows() + + +mock_inserted_rows = testing.make_mock_fixture( + bulk_upload, 'inserted_rows') + + +# relation_stats() + + +mock_relation_stats = testing.make_mock_fixture( + bulk_upload, 'relation_stats') + + +# log_success() + +csv_response = copy.deepcopy(CHANGED_RESPONSE) +csv_response['csv_checked'] = constants.CHECKED +tab_response = copy.deepcopy(CHANGED_RESPONSE) +tab_response['csv_checked'] = constants.UNCHECKED + + +@pytest.mark.parametrize( + ('response', 'stats', 'i_rows', 'r_stats'), [ + (csv_response, [], 1, []), + (tab_response, + [('dirA/foo', 2, 'foo'), ('dirB/foo', 3, 'foo')], + 3, + [('foo', 5)])]) +def test_log_success(caplog, response, stats, i_rows, r_stats): + '''A log message is output + ''' + caplog.set_level(logging.DEBUG) + + bulk_upload.log_success(response, stats, i_rows, r_stats) + + log_tuples = caplog.record_tuples + assert len(log_tuples) == 1 + assert (log_tuples[0][:2] + == ('pgwui_bulk_upload.views.bulk_upload', logging.INFO)) + + +mock_log_success = testing.make_mock_fixture( + bulk_upload, 'log_success') + + +# analyze_results() + + +def test_analyze_results_unchanged( + mock_log_success, mock_relation_stats, mock_inserted_rows, + mock_statmap, mock_stats, mock_quote_columns): + '''Response is returned unchanged, nothing is logged + ''' + response = UNCHANGED_RESPONSE + + uh = bulk_upload.BulkTableUploadHandler(response) + mocked_stats = mock_stats(uh) + mocked_quote_columns = mock_quote_columns(uh) + + result = bulk_upload.analyze_results(uh, response) + + assert mock_log_success.call_count == 0 + assert mock_relation_stats.call_count == 0 + assert mock_inserted_rows.call_count == 0 + assert mocked_stats.call_count == 0 + assert mocked_quote_columns.call_count == 0 + assert result == response + + +def test_analyze_results_changed( + mock_log_success, mock_relation_stats, mock_inserted_rows, + mock_statmap, mock_stats, mock_quote_columns): + '''Response is returned unchanged, nothing is logged + ''' + response = copy.deepcopy(CHANGED_RESPONSE) + + uh = bulk_upload.BulkTableUploadHandler(response) + mocked_stats = mock_stats(uh) + mocked_stats.return_value = [] + mocked_quote_columns = mock_quote_columns(uh) + + result = bulk_upload.analyze_results(uh, response) + + assert mock_log_success.call_count == 1 + assert mock_relation_stats.call_count == 1 + assert mock_inserted_rows.call_count == 1 + assert mocked_stats.call_count == 1 + assert mock_statmap.call_count == 1 + assert mocked_quote_columns.call_count == 1 + + assert 'stats' in result + assert 'inserted_rows' in result + assert 'relation_stats' in result + + +mock_analyze_results = testing.make_mock_fixture( + bulk_upload, 'analyze_results') + + +# Fixtures + +@pytest.fixture +def isolate_bulk_upload_view(monkeypatch, pyramid_request_config, + mock_analyze_results): + '''Keep bulk_upload_view() from calling anything + + Also, have isolate_bulk_upload_view(response) + cause UploadEngine to return the supplied "response". + ''' + def run(response): + def upload_engine(*args): + return MockUploadEngine(response) + + monkeypatch.setattr(bulk_upload, 'UploadEngine', upload_engine) + monkeypatch.setattr( + bulk_upload, 'BulkTableUploadHandler', MockBulkTableUploadHandler) + + settings = pyramid_request_config.get_settings() + settings['pgwui'] = settings.get('pgwui', dict()) + settings['pgwui'].update({'home_page': HOME_PAGE_SETTINGS}) + pgwui_common_includeme(pyramid_request_config) + pgwui_bulk_upload_includeme(pyramid_request_config) + settings['pgwui'].update({'urls': DEFAULT_URLS}) + pyramid_request_config.add_settings(settings) + + mock_analyze_results.return_value = response + + return run + + +# Tests + +# bulk_upload_view() + +def test_bulk_upload_view(isolate_bulk_upload_view): + '''The response contains and upload handler and + the request's bulk_upload settings + ''' + response = copy.deepcopy(UNCHANGED_RESPONSE) + isolate_bulk_upload_view(response) + request = get_current_request() + result = bulk_upload.bulk_upload_view(request) + + bulk_settings = request.registry.settings['pgwui']['pgwui_bulk_upload'] + assert result['pgwui']['pgwui_bulk_upload'] == bulk_settings + + del result['pgwui'] # Remove variables added by pgwui view decorators + assert result == response diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..b2fe965 --- /dev/null +++ b/tox.ini @@ -0,0 +1,37 @@ +[tox] +envlist = py{34,35,36,37} + +[testenv] +basepython = + py34: python3.4 + py35: python3.5 + py36: python3.6 + py37: python3.7 +deps = + check-manifest + cmarkgfm + flake8 + pgwui_testing + pytest + pytest-cov + twine + # coverage +commands = + check-manifest + python setup.py sdist + twine check dist/* + flake8 . + py.test -m unittest --cov=pgwui_bulk_upload tests/ + py.test -m 'not unittest' --cov=pgwui_bulk_upload tests/ + # coverage run --source src/pgwui_bulk_upload -m py.test + # coverage report + +[flake8] +exclude = .tox,*.egg,build,data,devel +select = E,W,F +ignore = W503 + +[pytest] +markers = + unittest + integrationtest -- 2.34.1