From ab4a21f1159be6259fceea50758825598cd31dee Mon Sep 17 00:00:00 2001 From: Faisal Shahzad <84210709+seowings@users.noreply.github.com> Date: Sun, 24 Sep 2023 01:30:40 +0200 Subject: [PATCH] converted to python package with GUI --- .gitignore | 39 +- LICENSE | 699 ++++++++++- MANIFEST.in | 1 + README.md | 4 +- docs/index.md | 2 +- mkdocs.yml | 7 +- netlify.toml | 2 +- pyproject.toml | 22 + requirements.txt | 3 + setup.cfg | 3 + setup.py | 154 +++ src/staticwordpress/__init__.py | 24 + src/staticwordpress/cli/__init__.py | 24 + src/staticwordpress/core/__init__.py | 37 + src/staticwordpress/core/constants.py | 151 +++ src/staticwordpress/core/crawler.py | 285 +++++ src/staticwordpress/core/github.py | 158 +++ src/staticwordpress/core/i18n.py | 73 ++ src/staticwordpress/core/project.py | 467 +++++++ src/staticwordpress/core/redirects.py | 129 ++ src/staticwordpress/core/search.py | 142 +++ src/staticwordpress/core/sitemaps.py | 101 ++ src/staticwordpress/core/utils.py | 225 ++++ src/staticwordpress/core/workflow.py | 387 ++++++ src/staticwordpress/gui/__init__.py | 30 + src/staticwordpress/gui/config.py | 435 +++++++ src/staticwordpress/gui/logger.py | 55 + src/staticwordpress/gui/mainwindow.py | 1069 +++++++++++++++++ src/staticwordpress/gui/rawtext.py | 81 ++ src/staticwordpress/gui/utils.py | 73 ++ src/staticwordpress/gui/workflow.py | 169 +++ src/staticwordpress/share/config.json | 713 +++++++++++ src/staticwordpress/share/gui.json | 428 +++++++ .../share/icons/additionals.svg | 38 + .../share/icons/bug-outline.svg | 36 + .../share/icons/check_project.svg | 38 + .../share/icons/close-box-outline.svg | 36 + .../share/icons/cloud-upload-outline.svg | 36 + src/staticwordpress/share/icons/configs.svg | 36 + .../share/icons/content-save-outline.svg | 36 + .../share/icons/crawl_website.svg | 38 + .../share/icons/delete-outline.svg | 38 + .../share/icons/delete-repo.svg | 38 + src/staticwordpress/share/icons/error.svg | 38 + .../share/icons/exit-to-app.svg | 36 + .../icons/file-document-remove-outline.svg | 36 + .../share/icons/file-outline.svg | 36 + .../share/icons/folder-git.svg | 59 + .../share/icons/folder-off.svg | 39 + .../share/icons/folder-outline.svg | 36 + src/staticwordpress/share/icons/github.svg | 36 + .../share/icons/help-box-outline.svg | 36 + .../share/icons/information-variant.svg | 36 + .../share/icons/pencil-outline.svg | 36 + src/staticwordpress/share/icons/person.svg | 38 + src/staticwordpress/share/icons/play.svg | 38 + .../share/icons/playlist-plus.svg | 36 + src/staticwordpress/share/icons/redirects.svg | 38 + .../share/icons/robots_txt.svg | 38 + src/staticwordpress/share/icons/search.svg | 38 + .../share/icons/semantic-web.svg | 1 + .../share/icons/static-wordpress.svg | 48 + src/staticwordpress/share/icons/stop.svg | 38 + .../share/icons/text-recognition.svg | 36 + .../share/icons/three-dots.svg | 38 + .../share/icons/web-remove.svg | 36 + src/staticwordpress/share/icons/wordpress.svg | 1 + src/staticwordpress/share/robots.txt | 3 + src/staticwordpress/share/search.js | 307 +++++ src/staticwordpress/share/translations.yaml | 6 + ss_script.py | 76 ++ tests/test_project.py | 38 + tests/test_redirects.py | 45 + tests/test_translations.py | 43 + tests/test_url.py | 51 + 75 files changed, 7941 insertions(+), 38 deletions(-) create mode 100644 MANIFEST.in create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/staticwordpress/__init__.py create mode 100644 src/staticwordpress/cli/__init__.py create mode 100644 src/staticwordpress/core/__init__.py create mode 100644 src/staticwordpress/core/constants.py create mode 100644 src/staticwordpress/core/crawler.py create mode 100644 src/staticwordpress/core/github.py create mode 100644 src/staticwordpress/core/i18n.py create mode 100644 src/staticwordpress/core/project.py create mode 100644 src/staticwordpress/core/redirects.py create mode 100644 src/staticwordpress/core/search.py create mode 100644 src/staticwordpress/core/sitemaps.py create mode 100644 src/staticwordpress/core/utils.py create mode 100644 src/staticwordpress/core/workflow.py create mode 100644 src/staticwordpress/gui/__init__.py create mode 100644 src/staticwordpress/gui/config.py create mode 100644 src/staticwordpress/gui/logger.py create mode 100644 src/staticwordpress/gui/mainwindow.py create mode 100644 src/staticwordpress/gui/rawtext.py create mode 100644 src/staticwordpress/gui/utils.py create mode 100644 src/staticwordpress/gui/workflow.py create mode 100644 src/staticwordpress/share/config.json create mode 100644 src/staticwordpress/share/gui.json create mode 100644 src/staticwordpress/share/icons/additionals.svg create mode 100644 src/staticwordpress/share/icons/bug-outline.svg create mode 100644 src/staticwordpress/share/icons/check_project.svg create mode 100644 src/staticwordpress/share/icons/close-box-outline.svg create mode 100644 src/staticwordpress/share/icons/cloud-upload-outline.svg create mode 100644 src/staticwordpress/share/icons/configs.svg create mode 100644 src/staticwordpress/share/icons/content-save-outline.svg create mode 100644 src/staticwordpress/share/icons/crawl_website.svg create mode 100644 src/staticwordpress/share/icons/delete-outline.svg create mode 100644 src/staticwordpress/share/icons/delete-repo.svg create mode 100644 src/staticwordpress/share/icons/error.svg create mode 100644 src/staticwordpress/share/icons/exit-to-app.svg create mode 100644 src/staticwordpress/share/icons/file-document-remove-outline.svg create mode 100644 src/staticwordpress/share/icons/file-outline.svg create mode 100644 src/staticwordpress/share/icons/folder-git.svg create mode 100644 src/staticwordpress/share/icons/folder-off.svg create mode 100644 src/staticwordpress/share/icons/folder-outline.svg create mode 100644 src/staticwordpress/share/icons/github.svg create mode 100644 src/staticwordpress/share/icons/help-box-outline.svg create mode 100644 src/staticwordpress/share/icons/information-variant.svg create mode 100644 src/staticwordpress/share/icons/pencil-outline.svg create mode 100644 src/staticwordpress/share/icons/person.svg create mode 100644 src/staticwordpress/share/icons/play.svg create mode 100644 src/staticwordpress/share/icons/playlist-plus.svg create mode 100644 src/staticwordpress/share/icons/redirects.svg create mode 100644 src/staticwordpress/share/icons/robots_txt.svg create mode 100644 src/staticwordpress/share/icons/search.svg create mode 100644 src/staticwordpress/share/icons/semantic-web.svg create mode 100644 src/staticwordpress/share/icons/static-wordpress.svg create mode 100644 src/staticwordpress/share/icons/stop.svg create mode 100644 src/staticwordpress/share/icons/text-recognition.svg create mode 100644 src/staticwordpress/share/icons/three-dots.svg create mode 100644 src/staticwordpress/share/icons/web-remove.svg create mode 100644 src/staticwordpress/share/icons/wordpress.svg create mode 100644 src/staticwordpress/share/robots.txt create mode 100644 src/staticwordpress/share/search.js create mode 100644 src/staticwordpress/share/translations.yaml create mode 100644 ss_script.py create mode 100644 tests/test_project.py create mode 100644 tests/test_redirects.py create mode 100644 tests/test_translations.py create mode 100644 tests/test_url.py diff --git a/.gitignore b/.gitignore index 376728e..dbd935f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ build/ develop-eggs/ dist/ downloads/ +versuche/ eggs/ .eggs/ lib/ @@ -20,7 +21,6 @@ parts/ sdist/ var/ wheels/ -pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +cover/ # Translations *.mo @@ -72,6 +73,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -82,7 +84,9 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -91,7 +95,22 @@ ipython_config.py # install all needed dependencies. #Pipfile.lock -# PEP 582; used by e.g. github.com/David-OConnor/pyflow +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff @@ -103,7 +122,6 @@ celerybeat.pid # Environments .env -.swp .venv env/ venv/ @@ -129,6 +147,15 @@ dmypy.json # Pyre type checker .pyre/ +# pytype static type analyzer +.pytype/ -# misc -pyvenv.cfg \ No newline at end of file +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/LICENSE b/LICENSE index f9c75a1..f288702 100644 --- a/LICENSE +++ b/LICENSE @@ -1,25 +1,674 @@ -Static Wordpress Netlify Process -https://github.com/serpwings/static-wordpress - -A Python Library to Prepare and Deploy Static version of WordPress Installation to -Static Hosting Service Providrs (Netlify). - - -MIT License -Copyright (c) 2023 SERP Wings - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..8f29ae9 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include src/staticwordpress/share/ *.* diff --git a/README.md b/README.md index c0983f3..afc2acc 100644 --- a/README.md +++ b/README.md @@ -21,5 +21,5 @@ This work is a collaborative effort of [seowings](https://seowings.org/) and [se ## LICENSE -- ``static-wordpress`` is released under [MIT License](https://github.com/serpwings/static-wordpress/blob/master/LICENSE). -- ``src\search.js`` is distributed without any additional licensing restrictions. Please consult ``src\search.js`` for more details. +- ``static-wordpress`` is released under [GPLv3+ License](https://github.com/serpwings/static-wordpress/blob/master/LICENSE). +- ``srcstaticwordpress\share\search.js`` is distributed without any additional licensing restrictions. Please consult ``src\search.js`` for more details. diff --git a/docs/index.md b/docs/index.md index 5b0b12e..0751fc5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,5 +17,5 @@ This work is a collaborative effort of [seowings](https://seowings.org/){:target ## LICENSE -- ``static-wordpress`` is released under [MIT License](https://github.com/serpwings/static-wordpress/blob/master/LICENSE). +- ``static-wordpress`` is released under [GPLv3+ License](https://github.com/serpwings/static-wordpress/blob/master/LICENSE). - ``src\search.js`` is distributed without any additional licensing restrictions. Please consult ``src\search.js`` for more details. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 726e850..c4a2b33 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,9 +1,10 @@ -site_name: static-wordpress +site_name: staticwordpress site_description: Python Package to Create, Manipulate and Analyze Website Sitemaps site_author: SERP Wings -site_url: https://simply-static.netlify.app -repo_url: https://github.com/serpwings/static-wordpress +site_url: https://static-wordpress-docs.netlify.app/ + +repo_url: https://github.com/serpwings/staticwordpress edit_uri: blob/main/docs/ nav: diff --git a/netlify.toml b/netlify.toml index f55cb09..929ac88 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,3 +1,3 @@ [build] publish = "output" - command = "python src/main.py" \ No newline at end of file + command = "python ss_script.py" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4b638a2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools", "wheel"] + +[tool.black] +target-version = ['py38'] +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + | env?? + | env +)/ +''' diff --git a/requirements.txt b/requirements.txt index 1776920..8dfce7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ beautifulsoup4==4.11.1 lxml>=4.9.1 requests==2.31.0 +GitPython==3.1.32 +PyGithub==1.59.1 +PyYAML==6.0.1 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f48fdad --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[metadata] +description-file = README.md +license_file = LICENSE diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f0e236a --- /dev/null +++ b/setup.py @@ -0,0 +1,154 @@ +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + setup.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import os +from setuptools import ( + find_packages, + setup, +) + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from src.staticwordpress.core import __version__ + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# DATABASE/CONSTANTS LIST +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +python_minor_min = 8 +python_minor_max = 11 +confirmed_python_versions = [ + "Programming Language :: Python :: 3.{MINOR:d}".format(MINOR=minor) + for minor in range(python_minor_min, python_minor_max + 1) +] + +# Fetch readme file +with open(os.path.join(os.path.dirname(__file__), "README.md")) as f: + long_description = f.read() + +# Define source directory (path) +SRC_DIR = "src" + +# Requirements for dev and gui +extras_require = { + "dev": [ + "black", + "python-language-server[all]", + "setuptools", + "twine", + "wheel", + "setuptools", + "pytest", + "pytest-cov", + "twine", + "wheel", + "mkdocs", + "mkdocs-gen-files", + "mkdocstrings[python]", + "pymdown-extensions", + ], + "gui": [ + "pyqt5", + ], +} +extras_require["all"] = list( + {rq for target in extras_require.keys() for rq in extras_require[target]} +) + +# Install package +setup( + name="staticwordpress", + packages=find_packages(SRC_DIR), + package_dir={"": SRC_DIR}, + version=__version__, + description="Python Package for Converting WordPress Installation to a Static Website", + long_description=long_description, + long_description_content_type="text/markdown", + author="Faisal Shahzad", + author_email="info@serpwings.com", + url="https://github.com/serpwings/staticwordpress", + download_url="https://github.com/serpwings/staticwordpress/archive/v%s.tar.gz" + % __version__, + license="GPLv3+", + keywords=[ + "wordpress", + "static-site-generators", + "search-engines", + "seo", + ], + scripts=[], + include_package_data=True, + python_requires=">=3.{MINOR:d}".format(MINOR=python_minor_min), + setup_requires=[], + install_requires=[ + "click", + "pyyaml", + "requests", + "beautifulsoup4", + "lxml", + "GitPython", + "PyGithub", + "PyYAML", + ], + extras_require=extras_require, + zip_safe=False, + entry_points={ + "console_scripts": [ + "staticwordpress = staticwordpress.gui:main", + ], + }, + classifiers=[ + "Development Status : 2 - Pre-Alpha", + "Environment :: Console", + "Environment :: X11 Applications", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: MacOS", + "Operating System :: POSIX :: BSD", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + ] + + confirmed_python_versions + + [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering", + "Topic :: System", + "Topic :: System :: Archiving", + "Topic :: System :: Archiving :: Backup", + "Topic :: System :: Archiving :: Mirroring", + "Topic :: System :: Filesystems", + "Topic :: System :: Systems Administration", + "Topic :: Utilities", + ], +) diff --git a/src/staticwordpress/__init__.py b/src/staticwordpress/__init__.py new file mode 100644 index 0000000..39ac7bf --- /dev/null +++ b/src/staticwordpress/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\__init__.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" diff --git a/src/staticwordpress/cli/__init__.py b/src/staticwordpress/cli/__init__.py new file mode 100644 index 0000000..59d4334 --- /dev/null +++ b/src/staticwordpress/cli/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\cli\__init__.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" diff --git a/src/staticwordpress/core/__init__.py b/src/staticwordpress/core/__init__.py new file mode 100644 index 0000000..48bd163 --- /dev/null +++ b/src/staticwordpress/core/__init__.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\__init__.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .constants import VERISON + + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +__version__ = VERISON diff --git a/src/staticwordpress/core/constants.py b/src/staticwordpress/core/constants.py new file mode 100644 index 0000000..8559c84 --- /dev/null +++ b/src/staticwordpress/core/constants.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\constants.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import re +import json +from pathlib import Path +from enum import Enum + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# CONSTANTS LIST +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +VERSION_MAJOR = 0 +VERSION_MINOR = 0 +VERSION_REVISION = 1 +VERISON = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_REVISION}" # -pre-alpha" + +SHARE_FOLDER_PATH = Path( + Path(__file__).resolve().parent, + "..", + "share", +) + +LINK_REGEX = re.compile( + "((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)", + re.DOTALL, +) + +CONFIG_PATH = SHARE_FOLDER_PATH / "config.json" +with CONFIG_PATH.open("r") as f: + CONFIGS = json.load(f) + + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +def save_configs(): + with CONFIG_PATH.open("w") as f: + json.dump(CONFIGS, f, indent=4) + + +class ExtendedEnum(Enum): + """An extended enum class to convert list of items in an enumration.""" + + @classmethod + def list(cls): + return list(map(lambda c: c.value, cls)) + + +class PROJECT(Enum): + """An enum for the different project operations.""" + + NEW = "NEW" + OPEN = "OPEN" + CLOSE = "CLOSE" + SAVED = "SAVED" + UPDATE = "UPDATE" + NOT_FOUND = "NOT_FOUND" + + +class HOST(ExtendedEnum): + """An enum for the different Hostings.""" + + NETLIFY = "NETLIFY" + # CLOUDFLARE = "CLOUDFLARE" + # LOCALHOST = "LOCALHOST" + + +class URL(ExtendedEnum): + """Supported URL types""" + + NONE = "NONE" + BINARY = "BINARY" + IMAGE = "IMAGE" + PDF = "PDF" + HTML = "HTML" + JS = "JS" + CSS = "CSS" + TXT = "TXT" # robots.txt + XML = "XML" # sitemap + JSON = "JSON" + FOLDER = "FOLDER" + HOME = "HOME" + FONTS = "FONTS" + ZIP = "ZIP" + + +class LANGUAGES(ExtendedEnum): + """Supported languages""" + + en_US = "en_US" + de_DE = "de_DE" + + +class SOURCE(ExtendedEnum): + """List of Data Sources""" + + CRAWL = "CRAWL" + ZIP = "ZIP" + + +class REDIRECTS(ExtendedEnum): + """Redirection Sources""" + + NONE = "NONE" # Do not include Redirects + REDIRECTION = "REDIRECTION" # redirects Plugin from WP Plugin Repository + + +class USER_AGENT(ExtendedEnum): + """Crawlers""" + + FIREFOX = "FIREFOX" + CHROME = "CHROME" + CUSTOM = "CUSTOM" + + +# Dict with enumeration mapping +ENUMS_MAP = { + "redirects": REDIRECTS, + "host": HOST, + "source": SOURCE, + "user-agent": USER_AGENT, +} diff --git a/src/staticwordpress/core/crawler.py b/src/staticwordpress/core/crawler.py new file mode 100644 index 0000000..992221e --- /dev/null +++ b/src/staticwordpress/core/crawler.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\crawler.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import re +import hashlib +import json +from urllib import parse +from pathlib import Path + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import requests +from requests import PreparedRequest +from requests.structures import CaseInsensitiveDict + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .utils import get_mock_response, get_remote_content, get_clean_url +from .constants import CONFIGS, URL, LINK_REGEX + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class Crawler: + def __init__(self, loc_: str, type_: URL = URL.FOLDER, scheme_="") -> None: + loc_ = parse.unquote(loc_).replace("\/", "/") + if not any([loc_.startswith(f"{scheme}://") for scheme in CONFIGS["SCHEMES"]]): + loc_ = f"{CONFIGS['DEFAULT_SCHEME']}://{loc_}" + + if CONFIGS["CLEAN"]["URL"]: + loc_ = get_clean_url(loc_, "", scheme_) + + self._type = type_ + self._loc = loc_ + self._urlparse = parse.urlparse(self._loc) + + file_ext = self._urlparse.path.split(".")[-1].upper() + if file_ext: + for keys in CONFIGS["FORMATS"]: + if file_ext in CONFIGS["FORMATS"][keys]: + self._type = URL[keys] + + if any( + [exclule_url in self._urlparse.path for exclule_url in CONFIGS["EXCLUDE"]] + ): + self._type = URL.NONE + + if self._type == URL.FOLDER: + self._loc = ( + f"{self._loc}{'/' if not self._urlparse.path.endswith('/') else ''}" + ) + self._urlparse = parse.urlparse(self._loc) + + self._response = get_mock_response(url_=self._urlparse) + self._internal_links = [] + self._externals_links = [] + self._hash = hashlib.sha256(self._loc.encode("utf-8")).hexdigest() + + @property + def hash(self) -> str: + return self._hash + + @property + def external_links(self) -> list: + return self._externals_links + + @property + def internal_links(self) -> list: + return self._internal_links + + @property + def status_code(self) -> int: + return self._response.status_code + + @property + def loc(self) -> str: + return self._loc + + @property + def scheme(self) -> str: + return self._urlparse.scheme + + @property + def netloc(self) -> str: + return self._urlparse.netloc + + @property + def path(self) -> str: + return self._urlparse.path + + @property + def params(self) -> str: + return self._urlparse.params + + @property + def fragment(self) -> str: + return self._urlparse.fragment + + @property + def content(self) -> bytes: + return self._response.content + + @property + def elapsed(self) -> float: + return self._response.elapsed + + @property + def encoding(self) -> str: + return self._response.encoding + + @property + def headers(self): # -> CaseInsensitiveDict[str]: + return self._response.headers + + @property + def history(self) -> list: + return self._response.history + + @property + def is_permanent_redirect(self) -> bool: + return self._response.is_permanent_redirect + + @property + def is_redirect(self) -> bool: + return self._response.is_redirect + + @property + def ok(self) -> bool: + return self._response.ok + + @property + def reason(self) -> str: + return self._response.reason + + @property + def request(self) -> PreparedRequest: + return self._response.request + + @property + def text(self) -> str: + return self._response.text + + @property + def url(self) -> str: + return self._response.url + + @property + def redirects_chain(self) -> list: + return [history.url for history in self._response.history] + + @property + def is_valid(self) -> bool: + return all( + [ + len(self._urlparse.scheme) > 0, + len(self._urlparse.netloc) > 0, + len(self._urlparse.netloc.split(".")) > 1, + len(self._urlparse.path) > 0 or self._type == URL.HOME, + ] + ) + + def update_scheme(self, new_schema: str = "https") -> None: + self._loc = self._loc.replace(self._urlparse.scheme, new_schema) + self._urlparse = parse.urlparse(self._loc) + self._hash = hashlib.sha256(self._loc.encode("utf-8")).hexdigest() + + def fetch(self) -> None: + if self.is_valid: + self._response = get_remote_content(self._urlparse) + + if self._type in [URL.FOLDER, URL.HTML, URL.JS, URL.HOME]: + extracted_urls = set( + [link[0] for link in re.findall(LINK_REGEX, self._response.text)] + ) + + self._internal_links = [ + url for url in extracted_urls if self._urlparse.netloc in url + ] + self._externals_links = [ + url for url in extracted_urls if self._urlparse.netloc not in url + ] + + def save(self, full_output_folder: Path, dst_url: str = "") -> str: + folder_path = ( + Path(self.path[1:]) if self.path.startswith("/") else Path(self.path) + ) + + full_output_path = Path(f"{full_output_folder}/{folder_path}") + + if self._response.status_code == 404: + self._type = URL.HTML + full_output_path = full_output_folder / Path("404.html") + + if self._type in [URL.FOLDER, URL.HOME]: + full_output_path = full_output_path / Path("index.html") + + if self._type not in [URL.NONE]: + full_output_path.parent.mkdir(parents=True, exist_ok=True) + + if self._type in [ + URL.HTML, + URL.XML, + URL.FOLDER, + URL.JS, + URL.CSS, + URL.TXT, + URL.HOME, + ]: + _text = self._response.text + if dst_url: + dest_url_parse = parse.urlparse(dst_url) + _text = self._response.text.replace( + f"{self._urlparse.scheme}://{self._urlparse.netloc}", + f"{dest_url_parse.scheme}://{dest_url_parse.netloc}", + ) + _text = _text.replace(self._urlparse.netloc, dest_url_parse.netloc) + + with open(full_output_path, "w", encoding="utf-8") as f: + f.write(_text) + + elif self._type in [URL.IMAGE, URL.PDF, URL.BINARY]: + with open(full_output_path, "wb") as file: + file.write(self._response.content) + + elif self._type in [URL.JSON]: + with open(full_output_path, "w", encoding="utf-8") as file: + json.dump(json.loads(self._response.text), file, indent=4) + + elif self._type == URL.ZIP: + headers = CaseInsensitiveDict() + headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + headers["Pragma"] = "no-cache" + headers["Expires"] = "0" + + current_session = requests.session() + response = current_session.get(self._loc, headers=headers) + + if response.status_code == 200: + with open(full_output_path, "wb") as fd: + for chunk in response.iter_content(chunk_size=128): + fd.write(chunk) + current_session.cookies.clear() + + elif self._type == URL.FONTS: + totalbits = 0 + if self._response.status_code == 200: + with open(full_output_path, "wb") as f: + for chunk in self._response.iter_content(chunk_size=1024): + if chunk: + totalbits += 1024 + f.write(chunk) + + return self._urlparse.path diff --git a/src/staticwordpress/core/github.py b/src/staticwordpress/core/github.py new file mode 100644 index 0000000..29a3be0 --- /dev/null +++ b/src/staticwordpress/core/github.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\github.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from datetime import datetime +from pathlib import Path +import logging + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import git +import github + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class GitHub: + def __init__(self, gh_token_: str, gh_repo_: str, repo_dir_: str) -> None: + assert gh_token_ != "" + assert gh_repo_ != "" + assert repo_dir_ != "" + + self._gh_token = gh_token_ + self._gh_repo = gh_repo_ + + if Path(f"{repo_dir_}/.git").exists(): + self._repo = git.Repo(repo_dir_) + else: + self._repo = git.Repo.init(repo_dir_) + + self._gh_object = github.Github(self._gh_token) + + # decorators + def check_gh_token(func): + def inner(self): + try: + self._gh_object.get_user().name + return func(self) + except: + pass + + return inner + + def check_repo_dir(func): + def inner(self): + try: + _ = self._repo.git_dir + return func(self) + except git.exc.InvalidGitRepositoryError: + pass + + return inner + + @check_gh_token + def is_token_valid(self): + logging.info(f"Verifying Github Token.") + return self._gh_object.get_user().name != "" + + @check_gh_token + def is_repo_valid(self): + logging.info(f"Verifying Github Repository.") + return self._gh_object.get_user().get_repo(self._gh_repo) is not None + + @check_gh_token + def create(self) -> None: + """create new repo at github""" + if self._gh_repo: + all_respos = [repo.name for repo in self._gh_object.get_user().get_repos()] + if self._gh_repo not in all_respos: + self._gh_object.get_user().create_repo(self._gh_repo, private=True) + logging.info( + f"Creating Remote Repository Successfully: {self._gh_repo}" + ) + + @check_gh_token + def delete(self) -> None: + """delete repo""" + if self._gh_repo and self._gh_object.get_user(): + all_respos = [repo.name for repo in self._gh_object.get_user().get_repos()] + if self._gh_repo in all_respos: + self._gh_object.get_user().get_repo(self._gh_repo).delete() + logging.info( + f"Deleting Remote Repository: {self._gh_repo} successfully" + ) + + @check_gh_token + def initialize(self) -> None: + """init repoinit repo""" + logging.info( + f"Initializing Git Repository - Orgins to GitHub will be set automatically" + ) + + login = self._gh_object.get_user().login + remote_url = ( + f"https://{login}:{self._gh_token}@github.com/{login}/{self._gh_repo}" + ) + + if self._repo.remotes: + origin = self._repo.remotes.origin + origin.set_url(remote_url, origin.url) + else: + origin = self._repo.create_remote("origin", url=remote_url) + + logging.info(f"Origin Exsitig for Git functionalities: {origin.exists()} ") + + assert origin.exists() + + logging.info(f"Updating Local Copy of Git Repository") + origin.fetch() + + @check_repo_dir + def commit(self) -> None: + """commit to local repository""" + logging.info(f"Start Committing Changes to Local Repository") + now = datetime.now() + date_time = now.strftime("%Y-%m-%d, %H:%M:%S") + self._repo.git.add("--all") + self._repo.index.commit(f"{date_time}: Update using static-wordpress software") + logging.info(f"All Changes Committed to Local Repository") + + @check_repo_dir + def publish(self): + """publish to github""" + self._repo.create_head("main") + if self._repo.remotes: + logging.info(f"Pushing Repository Changes to GitHub!") + self._repo.remotes[0].push("main") + else: + logging.error(f"Pushing Remote Repository on GitHub Failed.!") diff --git a/src/staticwordpress/core/i18n.py b/src/staticwordpress/core/i18n.py new file mode 100644 index 0000000..f69d936 --- /dev/null +++ b/src/staticwordpress/core/i18n.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\i18n.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import yaml +from pathlib import Path + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .constants import LANGUAGES, CONFIGS + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class _Translator(dict): + def __init__(self) -> None: + super().__init__() + self._lang = LANGUAGES[CONFIGS["LANGUAGE"]] + self._path = Path( + Path(__file__).resolve().parent, + "..", + "share", + "translations.yaml", + ) + + def __call__(self, index: str) -> str: + assert len(index) > 0 + return self.get(index, {}).get(self._lang.value, index) + + def load(self) -> None: + with open(self._path, "r", encoding="UTF-8") as f: + self.update(yaml.load(f.read(), Loader=yaml.CLoader)) + + @property + def language(self) -> LANGUAGES: + return self._lang + + @language.setter + def language(self, language_) -> None: + self._lang = language_ + + +tr = _Translator() +tr.load() diff --git a/src/staticwordpress/core/project.py b/src/staticwordpress/core/project.py new file mode 100644 index 0000000..d4f556a --- /dev/null +++ b/src/staticwordpress/core/project.py @@ -0,0 +1,467 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\project.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import re +import json +import base64 +import requests +from copy import deepcopy +from urllib import parse +from pathlib import Path, PosixPath, WindowsPath + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .constants import ( + PROJECT, + REDIRECTS, + HOST, + SOURCE, + CONFIGS, + VERISON, + USER_AGENT, + LINK_REGEX, +) + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class Project(dict): + def __init__(self, path_: str = None) -> None: + super().__init__() + + if isinstance(path_, str): + path_ = Path(path_) + + assert type(path_) == PosixPath or type(path_) == WindowsPath or path_ == None + + self["version"] = VERISON + self["path"] = path_ + self["status"] = PROJECT.NOT_FOUND + self["name"] = "" + self["scheme"] = CONFIGS["DEFAULT_SCHEME"] + self["user-agent"] = USER_AGENT.FIREFOX + self["source"] = { + "type": SOURCE.CRAWL, + "url": "", + "simply-static": { + "folder": CONFIGS["SIMPLYSTATIC"]["FOLDER"], + "archive": "", + }, + } + self["destination"] = { + "host": HOST.NETLIFY, + "url": "", + "output": "", + } + + self["wordpress"] = {"user": "", "api-token": ""} + self["github"] = {"token": "", "repository": ""} + self["redirects"] = REDIRECTS.REDIRECTION + self["sitemap"] = "sitemap_index.xml" + self["search"] = "search" + self["404"] = "404-error" + self["additional"] = [] + self["exclude"] = CONFIGS["EXCLUDE"] + self["delay"] = 0.1 + + def check_path_type(func): + def inner(self, path: str = None): + if isinstance(path, str): + path = Path(path) + return func(self, path) + + return inner + + def is_older_version(self) -> bool: + # TODO: compare major, minor and revision seperately + return self["version"] == VERISON + + def is_open(self) -> bool: + return all( + [ + self.status + in [ + PROJECT.OPEN, + PROJECT.SAVED, + PROJECT.UPDATE, + PROJECT.NEW, + ], + ] + ) + + def is_new(self) -> bool: + return self["status"] == PROJECT.NEW + + def is_saved(self) -> bool: + return self["status"] == PROJECT.SAVED + + def is_valid(self) -> bool: + return all( + [ + self["name"] != "", + self["source"]["url"] != "", + self["destination"]["output"] != "", + ] + ) + + def has_github(self) -> bool: + return self["github"]["token"] != "" or self["github"]["repository"] != "" + + def has_wordpress(self) -> bool: + return self["wordpress"]["api-token"] != "" or self["wordpress"]["user"] != "" + + def can_crawl(self) -> bool: + return all( + [ + self["source"]["type"] != SOURCE.CRAWL, + self["source"]["url"] != "", + self["destination"]["output"] != "", + ] + ) + + def create(self) -> None: + self["status"] = PROJECT.NEW + + def update_ss(self) -> None: + response = requests.get( + self.src_url + CONFIGS["SIMPLYSTATIC"]["API"], + headers={"Authorization": "Basic " + self.wp_auth_token}, + ) + + if response.status_code < 399: + ss_settings = json.loads((response).content) + ss_archive_urls = [ + url[0] + for url in re.findall( + LINK_REGEX, + ss_settings["archive_status_messages"]["create_zip_archive"][ + "message" + ], + ) + if url + ] + + if ss_archive_urls: + self["source"]["simply-static"]["folder"] = parse.urlparse( + ss_archive_urls[0] + ).path.split(ss_settings["archive_name"])[0] + + self["source"]["simply-static"]["archive"] = ss_settings["archive_name"] + + @check_path_type + def open(self, path_: str = None) -> None: + if path_ and isinstance(path_, Path): + self["path"] = path_ + + self["status"] = PROJECT.NOT_FOUND + if self["path"] and self["path"].exists(): + with self["path"].open("r") as f: + data = json.load(f) + if "version" not in data.keys(): + return + for key in self.keys(): + self[key] = data[key] + self["path"] = Path(data["path"]) + self["source"]["type"] = SOURCE[data["source"]["type"]] + self["user-agent"] = USER_AGENT[data["user-agent"]] + self["destination"]["host"] = HOST[data["destination"]["host"]] + self["destination"]["output"] = Path(data["destination"]["output"]) + self["redirects"] = REDIRECTS(data["redirects"]) + self["scheme"] = parse.urlparse(self["source"]["url"]).scheme + + # TODO: Check version and update to latest (if required) + self["status"] = PROJECT.SAVED + + @check_path_type + def save_as(self, path_: str = None) -> None: + if path_ and isinstance(path_, Path): + self["path"] = path_ + self.save() + + def save(self) -> None: + if self.is_open() and self["path"]: + with self["path"].open("w") as f: + _self_copy = deepcopy(self) + _self_copy["path"] = str(self["path"]) + _self_copy["user-agent"] = self["user-agent"].value + _self_copy["source"]["type"] = self["source"]["type"].value + _self_copy["destination"]["host"] = self["destination"]["host"].value + _self_copy["redirects"] = self["redirects"].value + _self_copy["destination"]["output"] = str(self["destination"]["output"]) + _self_copy["status"] = PROJECT.SAVED.value + + json.dump(_self_copy, f, indent=4) + + self["status"] = PROJECT.SAVED + + @property + def status(self) -> PROJECT: + return self["status"] + + @status.setter + def status(self, status_: PROJECT) -> None: + self["status"] = status_ + + @property + def scheme(self) -> str: + return self["scheme"] + + @property + def name(self) -> str: + return self["name"] + + @name.setter + def name(self, name_: str) -> None: + self["name"] = name_ + + @property + def path(self) -> Path: + return self["path"] + + @path.setter + def path(self, path_: str) -> None: + self["path"] = Path(path_) if isinstance(path_, str) else path_ + + @property + def version(self) -> str: + return self["version"] + + @property + def user_agent(self) -> str: + return self["user-agent"] + + @user_agent.setter + def user_agent(self, user_agent_: USER_AGENT) -> None: + self["user-agent"] = user_agent_ + + @property + def sitemap(self) -> str: + return self["sitemap"] + + @sitemap.setter + def sitemap(self, sitemap_: str) -> None: + self["sitemap"] = sitemap_ + + @property + def sitemap_url(self) -> str: + return f"{self.src_url}/{self['sitemap']}" + + @property + def destination(self) -> str: + return self["destination"] + + @destination.setter + def destination(self, destination_: dict) -> None: + self["destination"] = destination_ + + @property + def output(self) -> Path: + return self["destination"]["output"] + + @output.setter + def output(self, output_: Path) -> None: + self["destination"]["output"] = output_ + + @property + def dst_url(self) -> str: + return self["destination"]["url"] + + @dst_url.setter + def dst_url(self, dst_url_: str) -> None: + self["destination"]["url"] = dst_url_ + + @property + def host(self) -> str: + return self["destination"]["host"] + + @host.setter + def host(self, host_: HOST) -> None: + self["destination"]["host"] = host_ + + @property + def delay(self) -> float: + return self["delay"] + + @delay.setter + def delay(self, delay_: float) -> None: + self["delay"] = delay_ + + @property + def src_type(self) -> SOURCE: + return self["source"]["type"] + + @src_type.setter + def src_type(self, source_type_: str) -> None: + self["source"]["type"] = source_type_ + + @property + def src_url(self) -> str: + return self["source"]["url"] + + @src_url.setter + def src_url(self, src_url_: str) -> None: + self["source"]["url"] = src_url_ + + @property + def ss_archive(self) -> str: + return self["source"]["simply-static"]["archive"] + + @ss_archive.setter + def ss_archive(self, archive_name_: str) -> None: + self["source"]["simply-static"]["archive"] = archive_name_ + + @property + def ss_folder(self) -> str: + return self["source"]["simply-static"]["folder"] + + @ss_folder.setter + def ss_folder(self, folder_: str) -> None: + self["source"]["simply-static"]["folder"] = folder_ + + @property + def zip_file_url(self) -> str: + return f"{self.src_url}{self.ss_folder}{self.ss_archive}.zip" + + @property + def zip_file_path(self) -> Path: + return Path(f"{self.output}/{self.ss_folder}{self.ss_archive}.zip") + + @property + def wordpress(self) -> str: + return self["wordpress"] + + @wordpress.setter + def wordpress(self, wp_settings_: dict) -> None: + self["wordpress"] = wp_settings_ + + @property + def wp_user(self) -> str: + return self["wordpress"]["user"] + + @wp_user.setter + def wp_user(self, wp_user_: str) -> None: + self["wordpress"]["user"] = wp_user_ + + @property + def wp_api_token(self) -> str: + return self["wordpress"]["api-token"] + + @wp_api_token.setter + def wp_api_token(self, wp_api_token_: str) -> None: + self["wordpress"]["api-token"] = wp_api_token_ + + @property + def wp_auth_token(self) -> str: + if self.wp_user and self.wp_api_token: + return base64.b64encode( + f"{self.wp_user}:{self.wp_api_token}".encode() + ).decode("utf-8") + return "" + + @property + def github(self) -> str: + return self["github"] + + @property + def gh_repo(self) -> str: + return self["github"]["repository"] + + @gh_repo.setter + def gh_repo(self, gh_repo_: str) -> None: + self["github"]["repository"] = gh_repo_ + + @property + def gh_token(self) -> str: + return self["github"]["token"] + + @gh_token.setter + def gh_token(self, gh_token_: str) -> None: + self["github"]["token"] = gh_token_ + + @property + def additional(self) -> list: + return self["additional"] + + @additional.setter + def additional(self, additional_: list) -> None: + self["additional"] = [url for url in additional_ if url] + + @property + def exclude(self) -> list: + return self["exclude"] + + @exclude.setter + def exclude(self, exclude_: list) -> None: + self["exclude"] = [url for url in exclude_ if url] + + @property + def redirects(self) -> REDIRECTS: + return self["redirects"] + + @redirects.setter + def redirects(self, redirects_: REDIRECTS) -> None: + self["redirects"] = redirects_ + + @property + def redirects_api_url(self) -> str: + if self.redirects != REDIRECTS.NONE: + return f'{self.src_url}/{ CONFIGS["REDIRECTS"][self["redirects"].value]["API"]}' + return None + + @property + def search(self) -> str: + return self["search"] + + @search.setter + def search(self, search: str) -> None: + self["search"] = search + + @property + def search_path(self) -> Path: + return Path(f"{self.output}/{self['search']}") + + @property + def _404_path(self) -> Path: + return Path(f"{self.output}/{self['404']}") + + @property + def _404_url(self) -> str: + return f"{self.src_url}/{self['404']}" + + @property + def _404(self) -> str: + return self["404"] + + @_404.setter + def _404(self, page_404_) -> None: + self["404"] = page_404_ diff --git a/src/staticwordpress/core/redirects.py b/src/staticwordpress/core/redirects.py new file mode 100644 index 0000000..528f832 --- /dev/null +++ b/src/staticwordpress/core/redirects.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\redirects.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import json +import requests +import hashlib + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from ..core.constants import HOST, REDIRECTS + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class Redirect: + def __init__(self, from_, to_, query_, status, force_, source_) -> None: + self._from = from_ + self._to = to_ + self._query = query_ + self._status = status + self._force = force_ + self._source = source_ + self._hash = hashlib.sha256(from_.encode("utf-8")).hexdigest() + + @property + def hash(self): + return self._hash + + def as_json(self) -> dict: + return {"from": self._from, "to": self._to, "status": self._status} + + def as_line(self, new_line_: bool = False) -> str: + nl = "\n" if new_line_ else "" + return f"{self._from}\t{self._to}\t{self._status}{nl}" + + def as_toml(self) -> str: + # need cleanup here + if self._query: + return f'[[redirects]]\nfrom = "{self._from}"\nto = "{self._to}"\nquery = {self._query}\nstatus = {self._status}\nforce = {str(self._force).lower()}\n\n' + else: + return f'[[redirects]]\nfrom = "{self._from}"\nto = "{self._to}"\nstatus = {self._status}\nforce = {str(self._force).lower()}\n\n' + + +class Redirects: + def __init__(self) -> None: + self._items = dict() + + def add_redirect(self, redirect_: Redirect) -> None: + if redirect_.hash not in self._items: + self._items[redirect_.hash] = redirect_ + + def consolidate(self) -> None: + pass + + def add_redirects(self, redirects_list_: list) -> None: + for redirect in redirects_list_: + self.add_redirect(redirect) + + def save(self, output_file_, host_: HOST) -> None: + with open( + output_file_, + "w", + encoding="utf-8", + ) as f: + for _, redirect in self._items.items(): + if host_ == HOST.NETLIFY: + f.write(redirect.as_toml()) + else: + f.write(redirect.as_line(True)) + + def get_from_plugin(self, redirects_api_path: str, wp_auth_token_: str): + red_response = requests.get( + redirects_api_path, headers={"Authorization": "Basic " + wp_auth_token_} + ) + + redirects_dict = json.loads(red_response.content) + for red in redirects_dict["items"]: + self.add_redirect( + redirect_=Redirect( + from_=red["url"], + to_=red["action_data"]["url"], + status=red["action_code"], + query_=None, + force_=True, + source_=REDIRECTS.REDIRECTION.value, + ) + ) + + def add_search(self, search_page: str): + self.add_redirect( + Redirect( + from_="/*", + to_=f"/{search_page}/", + status=301, + query_='{s = ":s"}', + force_=True, + source_=REDIRECTS.NONE.value, + ) + ) diff --git a/src/staticwordpress/core/search.py b/src/staticwordpress/core/search.py new file mode 100644 index 0000000..0984547 --- /dev/null +++ b/src/staticwordpress/core/search.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\search.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import shutil +import json +from pathlib import Path, PurePosixPath + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from bs4 import BeautifulSoup +from bs4.formatter import HTMLFormatter + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from ..core.utils import get_clean_url, string_formatter +from ..core.constants import CONFIGS, SHARE_FOLDER_PATH + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class Search: + """Class to Geneate Search from HTML documents""" + + def __init__(self, search_page_: Path = None, dst_url_="") -> None: + """Intilialize with Path of Search Page as HTML + + Args: + search_page_ (Path, optional): Search Page Path. + dst_url_ (str, optional): Desitnation Url where Search Will be hosted". + """ + self._search_index = [] + self._search_path = search_page_ + self._search_path_lunr = Path(f"{search_page_}/lunr.json") + self._search_path_script = Path( + f"{search_page_}/{CONFIGS['SEARCH']['INDEX']['src']}" + ) + self._dst_url = dst_url_ + + def update(self, soup_: BeautifulSoup, output_path_: str) -> None: + """Update search page by adding new tags + + Args: + soup_ (BeautifulSoup): Soup of the HTML Page + output_path_ (str | Path): Save location for Search Index Page + """ + lunr_script_tag = [ + soup_.new_tag( + "script", + src=CONFIGS["LUNR"]["src"], + integrity=CONFIGS["LUNR"]["integrity"], + crossorigin=CONFIGS["LUNR"]["crossorigin"], + referrerpolicy=CONFIGS["LUNR"]["referrerpolicy"], + ), + soup_.new_tag( + "script", + src=CONFIGS["SEARCH"]["INDEX"]["src"], + ), + ] + + for script in lunr_script_tag: + soup_.find("head").append(str(script)) + + # TODO: In future add support for minification check + content = soup_.prettify(formatter=HTMLFormatter(string_formatter)) + + with open(output_path_, "w", encoding="utf-8") as f: + f.write(content) + + def add(self, soup_: BeautifulSoup, url_path_: str) -> None: + """Add new (as soup) page to search indexs + + Args: + soup_ (BeautifulSoup): Soup of the HTML Page + url_path_ (str): URL of the HTML page + make_text (bool): If True then Text from Page is included in Search + """ + clean_url = get_clean_url(self._dst_url, str(PurePosixPath(url_path_))) + title = soup_.find("title") + + if title and clean_url: + all_text_content = soup_.body.find_all(CONFIGS["SEARCH"]["HTML_TAGS"]) + + # TODO: need more cleaning here + search_text = " ".join( + [text for _t in all_text_content for text in _t.strings] + ) + + # update search index array + # TODO: include feature images + self._search_index.append( + { + "title": title.string, + "content": search_text + if CONFIGS["SEARCH"]["INCLUDE"]["CONTENT"] + else title.string, + "href": clean_url, + } + ) + + def copy_scripts(self): + """Copy Search.js into search folder""" + src = Path(f'{SHARE_FOLDER_PATH}/{CONFIGS["SEARCH"]["INDEX"]["src"]}') + if src.exists(): + shutil.copyfile(src, self._search_path_script) + + def save(self) -> None: + """Save Search Index as Json File into Lunr.json""" + if self._search_path.is_dir(): + with open(self._search_path_lunr, "w", encoding="utf-8") as fl: + json.dump(self._search_index, fl, indent=4, ensure_ascii=False) diff --git a/src/staticwordpress/core/sitemaps.py b/src/staticwordpress/core/sitemaps.py new file mode 100644 index 0000000..e7362c5 --- /dev/null +++ b/src/staticwordpress/core/sitemaps.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\sitemaps.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from bs4 import BeautifulSoup + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .utils import get_clean_url, get_remote_content +from .constants import CONFIGS +from urllib import parse + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +def find_sitemap_location(home_url: str) -> str: + """Finding Sitemap Location Using Home Url + + Args: + home_url (str): Source URL of the Website + + Returns: + str: Location of Sitemap + """ + for sitemap_path in CONFIGS["SITEMAP"]["SEARCH_PATHS"]: + sitemap_url = get_clean_url(home_url, sitemap_path) + response = get_remote_content(sitemap_url) + if response.status_code < 400: + return parse.urlparse(response.url).path + + # robots.txt + robots_txt = get_clean_url(home_url, "robots.txt") + response = get_remote_content(robots_txt) + if response: + for item in response.text.split("\n"): + if item.startswith("Sitemap:"): + return item.split("Sitemap:")[-1].strip() + + # check home page for link rel=sitemap + response = get_remote_content(home_url) + if response: + soup = BeautifulSoup(response.text, features="xml") + for link in soup.find_all("link"): + if link.has_attr("sitemap"): + return link["href"] + return "" + + +def extract_sitemap_paths(sitemap_url: str) -> list: + """Extract Sub-Sitemap from Index Sitemap + + Args: + sitemap_url (str): Index Sitemap Url + + Returns: + list: List of Sub-Sitemaps + """ + sitemap_paths = [] + response = get_remote_content(sitemap_url) + for item in response.text.split("\n"): + if ".xsl" in item: + st = item.find("//") + en = item.find(".xsl") + sitemap_paths.append(f"http:{item[st:en+4]}") + + soup = BeautifulSoup(response.text, features="xml") + if len(soup.find_all("sitemapindex")) > 0: + sitemap_paths += [ + sitemap.findNext("loc").text for sitemap in soup.find_all("sitemap") + ] + + return sitemap_paths diff --git a/src/staticwordpress/core/utils.py b/src/staticwordpress/core/utils.py new file mode 100644 index 0000000..15eac7a --- /dev/null +++ b/src/staticwordpress/core/utils.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\search.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import os +import re +import stat +import shutil +from urllib import parse +from pathlib import Path +from zipfile import ZipFile +from functools import lru_cache +from unittest.mock import Mock + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import requests +from requests.adapters import HTTPAdapter +from requests.models import Response +from bs4 import BeautifulSoup + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .constants import CONFIGS, LINK_REGEX + + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +def string_formatter(str): + """string formatter + + Args: + str (string): Input Strings + + Returns: + _type_: Resulted Strings + """ + return str + + +def rmtree_permission_error(func, path, exc_info): + """Deleteing folders/files after changing permissions. + + Args: + func (_type_): Sender Function + path (_type_): Path to be delete + exc_info (_type_): executable information + """ + os.chmod(path, stat.S_IWRITE) + os.unlink(path) + + +def get_clean_url(url_: str = "", path_: str = "", scheme_: str = "") -> str: + """Get Clean Url from URL and change Path of Final Url + + Args: + url_ (str, optional): Dirty Url. Defaults to "". + path_ (str, optional): Desired Path of the clean Url (default is Path of dirty url). + + Returns: + str: Clean Url with desired Path. + """ + if isinstance(url_, str): + url_ = parse.urlparse(url_) + + url_ = parse.urlunparse( + parse.ParseResult( + scheme=scheme_ if scheme_ else url_.scheme, + netloc=url_.netloc, + path=path_ if path_ else url_.path, + params="", + query="", + fragment="", + ) + ) + + url_ = url_.split(" None: + """Delte Directry tree at dir_path + + Args: + dir_path (Path | str, optional): Path/tree which need to be remvoed. + delete_root (bool, optional): If True then Parnt/root folder is not delted. + """ + if dir_path and isinstance(dir_path, str): + dir_path = Path(dir_path) + + if not dir_path.exists(): + return + + for _path in dir_path.glob("**/*"): + if _path.is_file(): + _path.unlink() + elif _path.is_dir(): + shutil.rmtree(_path, onerror=rmtree_permission_error) + + if delete_root: + dir_path.rmdir() + + +def get_mock_response(url_: str = None) -> Response: + """Genreate Mock HTTP Response + + Args: + url_ (str, optional): Input Url. Defaults to None. + + Returns: + Response: Mock response with 9999 as status code + """ + response = Mock(spec=Response) + response.text = "" + response.history = [] + response.status_code = 9999 + response.url = url_ + return response + + +@lru_cache +def get_remote_content(url_: parse.ParseResult, max_retires: int = 5) -> Response: + """Get remote content using request library + + Args: + url (str): url needed to be fetched + max_retires (int, optional): maximum tries to fetch the content. Defaults to 5. + Returns: + Response: request response object. + """ + url = get_clean_url(url_=url_) + try: + s = requests.Session() + s.mount(url, HTTPAdapter(max_retries=max_retires)) + default_user_agent = CONFIGS["DEFAULT_USER_AGENT"] + return s.get(url, headers=CONFIGS["HEADER"][default_user_agent]) + except: + return get_mock_response(url_=url) + + +def update_links(content: str, from_: str, to_: str) -> str: + """update links in content by replacing from_ urls with to_urls + + Args: + content (str): Content where links will be updated + from_ (str): Source Urls + to_ (str): Destination Url + + Returns: + str: _description_ + """ + from_ = parse.unquote(from_).replace("\/", "/") + to_ = parse.unquote(to_).replace("\/", "/") + from_ = from_[0:-1] + to_ = to_[0:-1] + return content.replace(from_, to_) + + +def extract_urls_from_raw_text(raw_text: str, dest_url: str, src_url: str) -> list: + """Extract Urls form a Raw Text using Regex + + Args: + raw_text (str): Text containing Urls + dest_url (str): Desitnation Url Path (required if an update is necessary) + src_url (str): Source Url Path (required if an update is necessary) + + Returns: + list: List of Urls extracted + """ + new_additional_links = [] + for link in re.findall(LINK_REGEX, raw_text): + item = link[0].replace("\/", "/").split("?")[0] + item = re.sub("\(|\)|\[|\]", "", item) + new_additional_link = update_links(item, dest_url, src_url) + + if new_additional_link and src_url in new_additional_link: + new_additional_links.append(new_additional_link) + + return new_additional_links + + +def extract_zip_file(zip_file_path: Path, output_location: Path) -> None: + """Extract ZipFile content to output_location Path + + Args: + zip_file_path (Path): Zip File Path + output_location (Path): Ouput File Path + """ + if output_location.is_dir() and zip_file_path.exists(): + with ZipFile(zip_file_path, "r") as zf: + zf.extractall(output_location) diff --git a/src/staticwordpress/core/workflow.py b/src/staticwordpress/core/workflow.py new file mode 100644 index 0000000..adf50dc --- /dev/null +++ b/src/staticwordpress/core/workflow.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\core\workflow.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import glob +import shutil +import codecs +import logging +import random +import time +from pathlib import Path + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import requests +from bs4 import BeautifulSoup + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .search import Search +from .github import GitHub +from .crawler import Crawler +from .project import Project +from .redirects import Redirects +from .sitemaps import find_sitemap_location, extract_sitemap_paths +from .utils import extract_zip_file, rm_dir_tree, update_links +from .constants import ( + CONFIGS, + SHARE_FOLDER_PATH, + URL, + HOST, + SOURCE, + PROJECT, + REDIRECTS, +) + + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class Workflow: + def __init__(self): + self._project = Project() + self._redirects = Redirects() + self._search = Search() + self._crawler = Crawler(loc_="", type_=URL.NONE) + self._urls = dict() + self._github = None + self._keep_running = True + + @property + def sitemap(self) -> str: + return self._project.sitemap + + def clear(self): + self._urls = dict() + + def create_project( + self, + project_name_: str = "", + project_path_: str = "", + wp_user_: str = "", + wp_api_token_: str = "", + src_url_: str = "", + dst_url_: str = "", + output_folder_: str = "", + custom_404_: str = "", + custom_search_: str = "", + src_type_: SOURCE = SOURCE.CRAWL, + host_type_: HOST = HOST.NETLIFY, + ) -> None: + self._project.status = PROJECT.NEW + self._project.name = project_name_ + self._project.path = project_path_ + self._project._404 = custom_404_ + self._project.search = custom_search_ + + self._project.wordpress = {"user": wp_user_, "api-token": wp_api_token_} + self._project.destination = { + "host": host_type_, + "url": dst_url_, + "output": output_folder_, + } + + self._project.src_type = src_type_ + self._project.src_url = src_url_ + + # TODO: Add Support for GH Repo ??? Do We need it? + if all( + [ + self._project.gh_token != "", + self._project.output != "", + self._project.gh_repo != "", + ] + ): + self._github = GitHub( + gh_token_=self._project.gh_token, + repo_dir_=self._project.output, + gh_repo_=self._project.gh_repo, + ) + + self._project.update_ss() + + def set_project(self, project_: Project) -> None: + self._project = project_ + + if all( + [ + self._project.gh_token != "", + self._project.output != "", + self._project.gh_repo != "", + ] + ): + self._github = GitHub( + gh_token_=self._project.gh_token, + repo_dir_=self._project.output, + gh_repo_=self._project.gh_repo, + ) + + if self._project.src_type == SOURCE.ZIP: + self._project.update_ss() + + def stop_calculations(self): + self._keep_running = False + logging.warn(f"Background Processings will Stop. Please wait!") + + def open_project(self): + pass + + def save_project(self): + pass + + def close_project(self): + pass + + def download_zip_file(self) -> None: + if self._keep_running: + rm_dir_tree(self._project.output, delete_root=False) + self._crawler = Crawler(loc_=self._project.zip_file_url, type_=URL.ZIP) + self._crawler.fetch() + self._crawler.save(full_output_folder=self._project.output) + + def setup_zip_folders(self) -> None: + if self._keep_running: + extract_zip_file( + self._project.zip_file_path, + output_location=Path(self._project.output), + ) + rm_dir_tree(Path(f"{self._project.output}/{self._project.ss_folder}")) + extracted_paths = glob.glob( + f"{self._project.output}/**/{self._project.ss_archive}", recursive=True + ) + if extracted_paths: + archive_folder = Path(extracted_paths[0]) + shutil.copytree( + archive_folder, self._project.output, dirs_exist_ok=True + ) + zip_download_folder = archive_folder.relative_to(self._project.output) + rm_dir_tree( + Path(f"{self._project.output}/{zip_download_folder.parts[0]}"), + delete_root=True, + ) + + def add_search(self) -> None: + """Now Process all folders with content/index.html files + only include html pages with content (blogs, pages)""" + if self._project.search_path.exists(): + self._search = Search( + search_page_=self._project.search_path, dst_url_=self._project.dst_url + ) + + for _path in glob.glob(f"{self._project.output}/**", recursive=True): + current_path = Path(_path) + if ( + all( + [ + exclude not in current_path.parts + for exclude in self._project.exclude + ] + ) + and current_path.parts[-1] == "index.html" + ): + if self._keep_running: + with codecs.open(current_path, "r", "utf-8") as f: + content = f.read() + content = update_links( + content, + self._project.src_url, + self._project.dst_url, + ) + + url_path = current_path.parent.relative_to( + self._project.output + ) + + soup = BeautifulSoup(content, "lxml") + if str(url_path) == self._project.search: + self._search.update( + soup_=soup, output_path_=current_path + ) + else: + self._search.add( + soup_=soup, + url_path_=url_path, + ) + + self._search.copy_scripts() + self._search.save() + + def add_redirects(self) -> None: + if self._keep_running: + if self._project.redirects != REDIRECTS.NONE: + self._redirects.get_from_plugin( + redirects_api_path=self._project.redirects_api_url, + wp_auth_token_=self._project.wp_auth_token, + ) + + if self._project.search_path.exists(): + self._redirects.add_search(search_page=self._project.search) + + redirect_ouputfile = f"{self._project.output}/{CONFIGS['REDIRECTS']['DESTINATION'][self._project.host.value]}" + self._redirects.save( + output_file_=redirect_ouputfile, host_=self._project.host + ) + + def add_robots_txt(self) -> None: + if self._keep_running: + src = Path(f"{SHARE_FOLDER_PATH}/robots.txt") + dst = Path(f"{self._project.output}/robots.txt") + + if src.exists(): + shutil.copyfile(src, dst) + + def add_404_page(self) -> None: + if self._keep_running: + self._crawler = Crawler( + loc_=self._project._404_url, + type_=URL.HTML, + scheme_=self._project.scheme, + ) + self._crawler.fetch() + self._crawler.save(full_output_folder=self._project.output) + + if self._project.src_type == SOURCE.ZIP: + if self._project._404_path.exists(): + shutil.copy2( + src=f"{self._project._404_path}/index.html", + dst=f"{self._project.output}/404.html", + ) + shutil.rmtree(self._project._404_path) + + # crawl Actions + def find_sitemap(self) -> None: + self._project.sitemap = find_sitemap_location(self._project.src_url) + + def crawl_sitemap(self) -> None: + if self._project.sitemap: + sitemap_paths = extract_sitemap_paths(sitemap_url=self._project.sitemap_url) + for sitemap_path in sitemap_paths: + if self._keep_running: + self.crawl_url(loc_=sitemap_path) + + def crawl_url(self, loc_): + current_url = Crawler(loc_=loc_, scheme_=self._project.scheme) + if current_url.hash not in self._urls: + current_url.fetch() + full_output_path = current_url.save( + self._project.output, dst_url=self._project.dst_url + ) + self._urls[current_url._hash] = current_url + + custom_message = "Saved" + if current_url.status_code >= 400 or current_url._type == URL.NONE: + custom_message = "Ignored" + + logging.info( + f"{custom_message}: {current_url.status_code} {current_url._type} {full_output_path}" + ) + + for internal_link in current_url.internal_links: + time.sleep(self._project.delay + random.random() / 100) + if self._keep_running: + self.crawl_url(internal_link) + + # Project Verifications + def verify_project_name(self): + logging.info(f"Verifying Project Name!") + return self._project.name != "" + + def verify_src_url(self): + logging.info(f"Verifying Source Url!") + current_url = Crawler(loc_=self._project.src_url, scheme_=self._project.scheme) + current_url.fetch() + return current_url.status_code < 399 # non error status codes + + def verify_output(self): + logging.info(f"Verifying Output Folder!") + return self._project.output.exists() + + def verify_wp_user(self): + logging.info(f"Verifying WordPress User Name!") + + response = requests.get( + self._project.redirects_api_url, + headers={"Authorization": "Basic " + self._project.wp_auth_token}, + ) + return response.status_code < 399 + + def verify_sitemap(self): + logging.info(f"Verifying Sitemap!") + + response = requests.get( + self._project.sitemap_url, + headers={"Authorization": "Basic " + self._project.wp_auth_token}, + ) + return response.status_code < 399 + + def verify_github_token(self): + return self._github.is_token_valid() + + def verify_github_repo(self): + return self._github.is_repo_valid() + + def verify_simply_static(self): + logging.info(f"Verifying simply static plugin!") + + response = requests.get( + self._project.src_url + CONFIGS["SIMPLYSTATIC"]["API"], + headers={"Authorization": "Basic " + self._project.wp_auth_token}, + ) + + ss_found = response.status_code < 399 + logging.info(f"Simply Static Plugin {'found' if ss_found else 'not found'}!") + logging.info("".join(140 * ["-"])) + return ss_found + + # Github Actions + def create_github_repositoy(self): + if self._keep_running: + self._github.create() + + def delete_github_repositoy(self): + if self._keep_running: + self._github.delete() + + def init_git_repositoy(self): + if self._keep_running: + self._github.initialize() + + def commit_git_repositoy(self): + if self._keep_running: + self._github.commit() + + def publish_github_repositoy(self): + if self._keep_running: + self._github.publish() diff --git a/src/staticwordpress/gui/__init__.py b/src/staticwordpress/gui/__init__.py new file mode 100644 index 0000000..b02053d --- /dev/null +++ b/src/staticwordpress/gui/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\gui\__init__.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from .mainwindow import main diff --git a/src/staticwordpress/gui/config.py b/src/staticwordpress/gui/config.py new file mode 100644 index 0000000..5f2f256 --- /dev/null +++ b/src/staticwordpress/gui/config.py @@ -0,0 +1,435 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\gui\config.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from PyQt5.QtWidgets import ( + QTabBar, + QStylePainter, + QStyleOptionTab, + QStyle, + QTabWidget, + QWidget, + QDialog, + QVBoxLayout, + QHBoxLayout, + QFormLayout, + QLineEdit, + QComboBox, + QDialogButtonBox, + QTextEdit, + QLabel, + QCheckBox, + QPushButton, + QGroupBox, + QColorDialog, +) +from PyQt5.QtCore import QSize, QPoint, QRect, QSettings +from PyQt5.QtGui import QPaintEvent, QIcon + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from ..core.constants import ( + LANGUAGES, + HOST, + REDIRECTS, + USER_AGENT, + CONFIGS, + SHARE_FOLDER_PATH, + save_configs, +) + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class ConfigTabBar(QTabBar): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + return + + def tabSizeHint(self, index: int) -> QSize: + s = super().tabSizeHint(index) + s.transpose() + return s + + def paintEvent(self, event: QPaintEvent) -> None: + painter: QStylePainter = QStylePainter(self) + opt: QStyleOptionTab = QStyleOptionTab() + for i in range(self.count()): + self.initStyleOption(opt, i) + painter.drawControl(QStyle.CE_TabBarTabShape, opt) + painter.save() + + s: QSize = opt.rect.size() + s.transpose() + r: QRect = QRect(QPoint(), s) + r.moveCenter(opt.rect.center()) + opt.rect = r + + c: QPoint = self.tabRect(i).center() + painter.translate(c) + painter.rotate(90) + painter.translate(-c) + painter.drawControl(QStyle.CE_TabBarTabLabel, opt) + painter.restore() + return + + +class ConfigWidget(QDialog): + def __init__(self): + super(ConfigWidget, self).__init__() + self.appConfigurations = QSettings( + CONFIGS["APPLICATION_NAME"], CONFIGS["APPLICATION_NAME"] + ) + + self.tabswidget_configs = QTabWidget() + self.tabswidget_configs.setTabBar(ConfigTabBar()) + self.tabswidget_configs.setTabPosition(QTabWidget.West) + + self.tab_general = QWidget() + vbox_layout_general = QVBoxLayout() + group_box_system = QGroupBox("System") + form_layout_system = QFormLayout() + group_box_system.setLayout(form_layout_system) + + vbox_layout_language = QHBoxLayout() + self.combobox_language = QComboBox() + self.combobox_language.setFixedWidth(80) + self.combobox_language.addItems([item.value for item in list(LANGUAGES)]) + vbox_layout_language.addWidget(self.combobox_language) + vbox_layout_language.addStretch() + vbox_layout_language.setContentsMargins(0, 0, 0, 0) + form_layout_system.addRow(QLabel("Language"), vbox_layout_language) + + vbox_layout_scheme = QHBoxLayout() + self.combobox_scheme = QComboBox() + self.combobox_scheme.setFixedWidth(80) + self.combobox_scheme.addItems(CONFIGS["SCHEMES"]) + self.combobox_scheme.setCurrentText(CONFIGS["DEFAULT_SCHEME"]) + vbox_layout_scheme.addWidget(self.combobox_scheme) + vbox_layout_scheme.addStretch() + vbox_layout_scheme.setContentsMargins(0, 0, 0, 0) + form_layout_system.addRow(QLabel("Scheme"), vbox_layout_scheme) + + self.checkbox_clean_url = QCheckBox("") + self.checkbox_clean_url.setChecked(CONFIGS["CLEAN"]["URL"]) + form_layout_system.addRow("Clean Url", self.checkbox_clean_url) + self.lineedit_clean_chars = QLineEdit(CONFIGS["CLEAN"]["CHARS"]) + form_layout_system.addRow("Clean Chars", self.lineedit_clean_chars) + + group_box_colors = QGroupBox("Colors") + form_layout_colors = QFormLayout() + group_box_colors.setLayout(form_layout_colors) + + vbox_layout_color_success = QHBoxLayout() + self.pushbutton_color_success = QPushButton("") + self.pushbutton_color_success.setObjectName("success") + self.pushbutton_color_success.setFixedWidth(80) + self.pushbutton_color_success.clicked.connect(self.openColorDialog) + self.pushbutton_color_success.setStyleSheet( + f"background-color: {CONFIGS['COLOR']['SUCCESS']}" + ) + vbox_layout_color_success.addWidget(self.pushbutton_color_success) + vbox_layout_color_success.addStretch() + form_layout_colors.addRow("Success", vbox_layout_color_success) + + vbox_layout_color_warning = QHBoxLayout() + self.pushbutton_color_warning = QPushButton("") + self.pushbutton_color_warning.setObjectName("warning") + self.pushbutton_color_warning.setFixedWidth(80) + self.pushbutton_color_warning.clicked.connect(self.openColorDialog) + self.pushbutton_color_warning.setStyleSheet( + f"background-color: {CONFIGS['COLOR']['WARNING']}" + ) + vbox_layout_color_warning.addWidget(self.pushbutton_color_warning) + vbox_layout_color_warning.addStretch() + form_layout_colors.addRow("Warning", vbox_layout_color_warning) + + vbox_layout_color_error = QHBoxLayout() + self.pushbotton_color_error = QPushButton("") + self.pushbotton_color_error.setObjectName("error") + self.pushbotton_color_error.setFixedWidth(80) + self.pushbotton_color_error.clicked.connect(self.openColorDialog) + self.pushbotton_color_error.setStyleSheet( + f"background-color: {CONFIGS['COLOR']['ERROR']}" + ) + vbox_layout_color_error.addWidget(self.pushbotton_color_error) + vbox_layout_color_error.addStretch() + form_layout_colors.addRow("Error", vbox_layout_color_error) + + group_box_search = QGroupBox("Search") + form_layout_search = QFormLayout() + + layout_search_include = QHBoxLayout() + self.checkbox_search_content = QCheckBox("Textual Content") + self.checkbox_search_content.setChecked(CONFIGS["SEARCH"]["INCLUDE"]["CONTENT"]) + self.checkbox_search_iamge = QCheckBox("Feature Image") + self.checkbox_search_iamge.setChecked(CONFIGS["SEARCH"]["INCLUDE"]["IMAGE"]) + layout_search_include.addWidget(self.checkbox_search_content) + layout_search_include.addWidget(self.checkbox_search_iamge) + layout_search_include.addStretch() + form_layout_search.addRow("Include", layout_search_include) + self.lineedit_search_html_tags = QLineEdit( + ",".join(CONFIGS["SEARCH"]["HTML_TAGS"]) + ) + form_layout_search.addRow("HTML Tags", self.lineedit_search_html_tags) + + self.lineedit_search_src = QLineEdit(CONFIGS["SEARCH"]["INDEX"]["src"]) + form_layout_search.addRow("Source File", self.lineedit_search_src) + self.lineedit_lunar_src = QLineEdit(CONFIGS["LUNR"]["src"]) + form_layout_search.addRow("Lunr JS File", self.lineedit_lunar_src) + self.lineedit_lunar_integrity = QLineEdit(CONFIGS["LUNR"]["integrity"]) + form_layout_search.addRow("Lunr Integrity", self.lineedit_lunar_integrity) + group_box_search.setLayout(form_layout_search) + + vbox_layout_general.addWidget(group_box_system) + vbox_layout_general.addWidget(group_box_colors) + vbox_layout_general.addWidget(group_box_search) + vbox_layout_general.addStretch() + self.tab_general.setLayout(vbox_layout_general) + self.tabswidget_configs.addTab(self.tab_general, "General") + + self.tab_crawl = QWidget() + vbox_layout_crawl = QVBoxLayout() + + group_box_user_agent = QGroupBox("User Agents") + form_layout_user_agent = QFormLayout() + self.lineedit_user_agent_firefrox = QLineEdit( + CONFIGS["HEADER"]["FIREFOX"]["User-Agent"] + ) + form_layout_user_agent.addRow("FireFox", self.lineedit_user_agent_firefrox) + self.lineedit_user_agent_chrome = QLineEdit( + CONFIGS["HEADER"]["CHROME"]["User-Agent"] + ) + form_layout_user_agent.addRow("Chrome", self.lineedit_user_agent_chrome) + self.lineedit_user_agent_custom = QLineEdit( + CONFIGS["HEADER"]["CUSTOM"]["User-Agent"] + ) + form_layout_user_agent.addRow("Custom", self.lineedit_user_agent_custom) + self.combobox_default_user_agent = QComboBox() + self.combobox_default_user_agent.setFixedWidth(120) + self.combobox_default_user_agent.addItems( + [item.value for item in list(USER_AGENT)] + ) + self.combobox_default_user_agent.setCurrentText(CONFIGS["DEFAULT_USER_AGENT"]) + form_layout_user_agent.addRow("Default", self.combobox_default_user_agent) + group_box_user_agent.setLayout(form_layout_user_agent) + + group_box_redirects = QGroupBox("Redirects") + form_layout_redirects = QFormLayout() + + vbox_layout_redirects_plugins = QHBoxLayout() + self.combobox_redirects_plugin = QComboBox() + self.combobox_redirects_plugin.setFixedWidth(120) + self.combobox_redirects_plugin.addItems( + [item.value for item in list(REDIRECTS)] + ) + vbox_layout_redirects_plugins.addWidget(self.combobox_redirects_plugin) + vbox_layout_redirects_plugins.addStretch() + vbox_layout_redirects_plugins.setContentsMargins(0, 0, 0, 0) + form_layout_redirects.addRow("Plugin", vbox_layout_redirects_plugins) + group_box_redirects.setLayout(form_layout_redirects) + + self.lineedit_redirection_api = QLineEdit( + CONFIGS["REDIRECTS"][self.combobox_redirects_plugin.currentText()]["API"] + ) + self.combobox_redirects_plugin.currentIndexChanged.connect( + self.update_redirects_api + ) + form_layout_redirects.addRow("API", self.lineedit_redirection_api) + + group_box_simply_static = QGroupBox("Simply Static") + form_layout_simply_static = QFormLayout() + self.lineedit_simply_static_api = QLineEdit(CONFIGS["SIMPLYSTATIC"]["API"]) + form_layout_simply_static.addRow("API", self.lineedit_simply_static_api) + self.lineedit_simply_static_folder = QLineEdit( + CONFIGS["SIMPLYSTATIC"]["FOLDER"] + ) + form_layout_simply_static.addRow("Path", self.lineedit_simply_static_folder) + group_box_simply_static.setLayout(form_layout_simply_static) + + group_box_exclude = QGroupBox("Exclude") + vbox_layout_exclude = QVBoxLayout() + self.textedit_exclude = QTextEdit() + self.textedit_exclude.setObjectName("exclude") + self.textedit_exclude.setText("\n".join(CONFIGS["EXCLUDE"])) + vbox_layout_exclude.addWidget(self.textedit_exclude) + group_box_exclude.setLayout(vbox_layout_exclude) + + vbox_layout_crawl.addWidget(group_box_user_agent) + vbox_layout_crawl.addWidget(group_box_redirects) + vbox_layout_crawl.addWidget(group_box_simply_static) + vbox_layout_crawl.addWidget(group_box_exclude) + self.tab_crawl.setLayout(vbox_layout_crawl) + self.tabswidget_configs.addTab(self.tab_crawl, "Crawl") + + self.tab_sitemaps = QWidget() + sitemap_layout = QVBoxLayout() + self.checkbox_auto_sitemap_finder = QCheckBox("Auto Sitemap Finder") + self.checkbox_auto_sitemap_finder.setChecked(CONFIGS["SITEMAP"]["AUTO"]) + sitemap_layout.addWidget(self.checkbox_auto_sitemap_finder) + sitemap_layout.addWidget(QLabel("Possible Sitemap Locations")) + self.textedit_sitemap_locations = QTextEdit() + self.textedit_sitemap_locations.setObjectName("sitemap-locations") + self.textedit_sitemap_locations.setText( + "\n".join(CONFIGS["SITEMAP"]["SEARCH_PATHS"]) + ) + sitemap_layout.addWidget(self.textedit_sitemap_locations) + self.tab_sitemaps.setLayout(sitemap_layout) + self.tabswidget_configs.addTab(self.tab_sitemaps, "Sitemaps") + + # formats + self.tab_data_formats = QWidget() + data_format_layout = QVBoxLayout() + + group_box_image_formats = QGroupBox("Image Formats") + vbox_layout_image_formats = QVBoxLayout() + self.textedit_image_formats = QTextEdit() + self.textedit_image_formats.setObjectName("image-foramts") + self.textedit_image_formats.setText("\n".join(CONFIGS["FORMATS"]["IMAGE"])) + vbox_layout_image_formats.addWidget(self.textedit_image_formats) + group_box_image_formats.setLayout(vbox_layout_image_formats) + data_format_layout.addWidget(group_box_image_formats) + + group_box_font_formats = QGroupBox("Font Formats") + vbox_layout_font_formats = QVBoxLayout() + self.textedit_font_formats = QTextEdit() + self.textedit_font_formats.setObjectName("image-foramts") + self.textedit_font_formats.setText("\n".join(CONFIGS["FORMATS"]["FONTS"])) + vbox_layout_font_formats.addWidget(self.textedit_font_formats) + group_box_font_formats.setLayout(vbox_layout_font_formats) + data_format_layout.addWidget(group_box_font_formats) + + self.tab_data_formats.setLayout(data_format_layout) + self.tabswidget_configs.addTab(self.tab_data_formats, "File Formats") + + # robots.txt + with open(f"{SHARE_FOLDER_PATH}/robots.txt", "r") as f: + robots_txt_content = f.read() + + self.tab_robots_txt = QWidget() + vbox_layout_robots_format = QVBoxLayout() + self.textedit_robots_txt = QTextEdit() + self.textedit_robots_txt.setObjectName("robots-txt") + self.textedit_robots_txt.setText(robots_txt_content) + vbox_layout_robots_format.addWidget(self.textedit_robots_txt) + self.tab_robots_txt.setLayout(vbox_layout_robots_format) + self.tabswidget_configs.addTab(self.tab_robots_txt, "Robots.txt") + + dialog_button_box = QDialogButtonBox( + QDialogButtonBox.Ok | QDialogButtonBox.Cancel + ) + dialog_button_box.accepted.connect(self.accept) + dialog_button_box.rejected.connect(self.reject) + + self.layout = QVBoxLayout(self) + self.layout.setSpacing(3) + self.layout.setContentsMargins(1, 3, 3, 3) + + self.layout.addWidget(self.tabswidget_configs) + self.layout.addWidget(dialog_button_box) + self.setLayout(self.layout) + self.setFixedSize(QSize(620, 480)) + self.setWindowTitle("Default Configurations") + self.setWindowIcon(QIcon(f"{SHARE_FOLDER_PATH}/icons/static-wordpress.svg")) + + def openColorDialog(self): + color = QColorDialog.getColor() + + if color.isValid(): + self.sender().setStyleSheet(f"background-color: {color.name()}") + CONFIGS["COLOR"][self.sender().objectName().upper()] = color.name() + + def update_redirects_api(self): + self.lineedit_redirection_api.setText( + CONFIGS["REDIRECTS"][self.combobox_redirects_plugin.currentText()]["API"] + ) + + def accept(self) -> None: + # General + CONFIGS["LANGUAGE"] = self.combobox_language.currentText() + CONFIGS["DEFAULT_SCHEME"] = self.combobox_scheme.currentText() + CONFIGS["CLEAN"]["URL"] = self.checkbox_clean_url.isChecked() + CONFIGS["CLEAN"]["CHARS"] = self.lineedit_clean_chars.text() + CONFIGS["SEARCH"]["INDEX"]["src"] = self.lineedit_search_src.text() + CONFIGS["SEARCH"]["INCLUDE"][ + "CONTENT" + ] = self.checkbox_search_content.isChecked() + CONFIGS["SEARCH"]["INCLUDE"]["IMAGE"] = self.checkbox_search_iamge.isChecked() + CONFIGS["SEARCH"]["HTML_TAGS"] = self.lineedit_search_html_tags.text().split( + "," + ) + CONFIGS["LUNR"]["src"] = self.lineedit_lunar_src.text() + CONFIGS["LUNR"]["integrity"] = self.lineedit_lunar_integrity.text() + + # Crawl + redirect_type = self.combobox_redirects_plugin.currentText() + CONFIGS["REDIRECTS"][redirect_type][ + "API" + ] = self.lineedit_redirection_api.text() + + CONFIGS["SIMPLYSTATIC"]["API"] = self.lineedit_simply_static_api.text() + CONFIGS["SIMPLYSTATIC"]["FOLDER"] = self.lineedit_simply_static_folder.text() + CONFIGS["HEADER"]["CHROME"][ + "User-Agent" + ] = self.lineedit_user_agent_chrome.text() + CONFIGS["HEADER"]["CUSTOM"][ + "User-Agent" + ] = self.lineedit_user_agent_custom.text() + CONFIGS["HEADER"]["FIREFOX"][ + "User-Agent" + ] = self.lineedit_user_agent_firefrox.text() + + CONFIGS["DEFAULT_USER_AGENT"] = self.combobox_default_user_agent.currentText() + CONFIGS["EXCLUDE"] = self.textedit_exclude.toPlainText().split("\n") + + # Sitemap + CONFIGS["SITEMAP"]["AUTO"] = self.checkbox_auto_sitemap_finder.isChecked() + CONFIGS["SITEMAP"][ + "SEARCH_PATHS" + ] = self.textedit_sitemap_locations.toPlainText().split("\n") + + # file formats + CONFIGS["FORMATS"]["IMAGE"] = self.textedit_image_formats.toPlainText().split( + "\n" + ) + CONFIGS["FORMATS"]["FONTS"] = self.textedit_font_formats.toPlainText().split( + "\n" + ) + + # save/update configs + save_configs() + + # robots.txt save + with open(f"{SHARE_FOLDER_PATH}/robots.txt", "w") as f: + f.write(self.textedit_robots_txt.toPlainText()) + + return super().accept() diff --git a/src/staticwordpress/gui/logger.py b/src/staticwordpress/gui/logger.py new file mode 100644 index 0000000..706ccc9 --- /dev/null +++ b/src/staticwordpress/gui/logger.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\gui\logger.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import logging + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from PyQt5.QtWidgets import QPlainTextEdit +from PyQt5.QtCore import QObject, pyqtSignal + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class LoggerWidget(logging.Handler, QObject): + appendPlainText = pyqtSignal(str) + + def __init__(self, parent): + super().__init__() + QObject.__init__(self) + self.plainTextEdit = QPlainTextEdit(parent) + self.plainTextEdit.setReadOnly(True) + self.appendPlainText.connect(self.plainTextEdit.appendPlainText) + + def emit(self, msg): + self.appendPlainText.emit(self.format(msg)) diff --git a/src/staticwordpress/gui/mainwindow.py b/src/staticwordpress/gui/mainwindow.py new file mode 100644 index 0000000..0ae9be6 --- /dev/null +++ b/src/staticwordpress/gui/mainwindow.py @@ -0,0 +1,1069 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\gui\mainwindow.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import sys +import logging +import os +from pathlib import Path +from datetime import date + + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from PyQt5.QtWidgets import ( + QLabel, + QLineEdit, + QMainWindow, + QTextEdit, + QVBoxLayout, + QHBoxLayout, + QToolButton, + QAction, + QApplication, + QDockWidget, + QWidget, + QFileDialog, + QMessageBox, + QComboBox, + QFormLayout, + QProgressBar, + QMenu, + QToolBar, +) +from PyQt5.QtGui import QIcon, QDesktopServices +from PyQt5.QtCore import Qt, QThread, QSize, QSettings, QUrl + + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from ..core.constants import ( + VERISON, + CONFIGS, + ENUMS_MAP, + SHARE_FOLDER_PATH, + HOST, + REDIRECTS, + PROJECT, + SOURCE, + USER_AGENT, +) +from ..core.project import Project +from ..core.utils import ( + rm_dir_tree, + get_remote_content, + extract_urls_from_raw_text, +) +from .workflow import WorkflowGUI +from ..gui.logger import LoggerWidget +from ..gui.rawtext import RawTextWidget +from ..gui.config import ConfigWidget +from ..gui.utils import logging_decorator, GUI_SETTINGS + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + + +class StaticWordPressGUI(QMainWindow): + def __init__(self): + super().__init__() + self.appConfigurations = QSettings( + CONFIGS["APPLICATION_NAME"], CONFIGS["APPLICATION_NAME"] + ) + self.appConfigurations.setValue("icon_path", SHARE_FOLDER_PATH) + + self._project = Project() + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + + self.docked_widget_project_properties = QDockWidget("Project Properties", self) + self.docked_widget_project_properties.setMinimumSize(QSize(400, 100)) + + widget_project_properties = QWidget() + form_layout_project_properties = QFormLayout() + + self.lineedit_project_name = QLineEdit() + self.lineedit_project_name.setObjectName("name") + self.lineedit_project_name.textChanged.connect(self.update_windows_title) + form_layout_project_properties.addRow( + QLabel("Project Name:"), self.lineedit_project_name + ) + + horizontal_Layout_project_scource = QHBoxLayout() + self.combobox_source_type = QComboBox() + self.combobox_source_type.setObjectName("source") + self.combobox_source_type.currentTextChanged.connect(self.update_windows_title) + self.combobox_source_type.setMinimumWidth(120) + self.combobox_source_type.addItems([item.value for item in list(SOURCE)]) + horizontal_Layout_project_scource.addWidget(self.combobox_source_type) + horizontal_Layout_project_scource.addStretch() + self.combobox_user_agent = QComboBox() + self.combobox_user_agent.setObjectName("user-agent") + self.combobox_user_agent.setMinimumWidth(120) + self.combobox_user_agent.addItems([item.value for item in list(USER_AGENT)]) + self.combobox_user_agent.currentTextChanged.connect(self.update_windows_title) + horizontal_Layout_project_scource.addWidget(QLabel("User Agent")) + horizontal_Layout_project_scource.addWidget(self.combobox_user_agent) + form_layout_project_properties.addRow( + QLabel("Project Source"), horizontal_Layout_project_scource + ) + self.lineedit_src_url = QLineEdit() + self.lineedit_src_url.setObjectName("src-url") + self.lineedit_src_url.textChanged.connect(self.update_windows_title) + form_layout_project_properties.addRow( + QLabel("Source Url"), self.lineedit_src_url + ) + self.combobox_project_destination = QComboBox() + self.combobox_project_destination.setObjectName("host") + self.combobox_project_destination.currentTextChanged.connect( + self.update_windows_title + ) + + self.combobox_project_destination.addItems([item.value for item in list(HOST)]) + form_layout_project_properties.addRow( + QLabel("Destination Host"), self.combobox_project_destination + ) + + self.lineedit_dest_url = QLineEdit() + self.lineedit_dest_url.setObjectName("dst-url") + self.lineedit_dest_url.textChanged.connect(self.update_windows_title) + form_layout_project_properties.addRow( + QLabel("Destination Url"), self.lineedit_dest_url + ) + + horizontal_Layout_output_directory = QHBoxLayout() + self.lineedit_output = QLineEdit() + self.lineedit_output.setObjectName("output") + self.lineedit_output.textChanged.connect(self.update_windows_title) + self.toolbutton_output_directory = QToolButton() + self.toolbutton_output_directory.setIcon( + QIcon(f"{SHARE_FOLDER_PATH}/icons/three-dots.svg") + ) + horizontal_Layout_output_directory.addWidget(self.lineedit_output) + horizontal_Layout_output_directory.addWidget(self.toolbutton_output_directory) + self.toolbutton_output_directory.clicked.connect(self.get_output_directory) + form_layout_project_properties.addRow( + QLabel("Output Directory"), horizontal_Layout_output_directory + ) + + vertical_layout_additional_properties = QVBoxLayout() + vertical_layout_additional_properties.addLayout(form_layout_project_properties) + widget_project_properties.setLayout(vertical_layout_additional_properties) + self.docked_widget_project_properties.setWidget(widget_project_properties) + self.docked_widget_project_properties.setFeatures( + QDockWidget.NoDockWidgetFeatures + ) + + self.docked_widget_project_properties.setMaximumHeight(200) + self.addDockWidget(Qt.LeftDockWidgetArea, self.docked_widget_project_properties) + + # ============================= + # Github Properties dock + # ============================= + self.docked_widget_github_properties = QDockWidget("Github Setttings", self) + self.docked_widget_github_properties.setMaximumHeight(100) + + widget_github_properties = QWidget() + form_layout_github_properties = QFormLayout() + + self.lineedit_gh_repo = QLineEdit() + self.lineedit_gh_repo.setObjectName("github-repository") + self.lineedit_gh_repo.textChanged.connect(self.update_windows_title) + form_layout_github_properties.addRow( + QLabel("Repository Name"), self.lineedit_gh_repo + ) + self.lineedit_gh_token = QLineEdit() + self.lineedit_gh_token.setObjectName("github-token") + self.lineedit_gh_token.textChanged.connect(self.update_windows_title) + form_layout_github_properties.addRow( + QLabel("GitHub Token"), self.lineedit_gh_token + ) + + vertical_layout_github_properties = QVBoxLayout() + vertical_layout_github_properties.addLayout(form_layout_github_properties) + widget_github_properties.setLayout(vertical_layout_github_properties) + self.docked_widget_github_properties.setWidget(widget_github_properties) + self.docked_widget_github_properties.setFeatures( + QDockWidget.NoDockWidgetFeatures + ) + self.addDockWidget(Qt.LeftDockWidgetArea, self.docked_widget_github_properties) + + # ============================= + # Crawl Properties dock + # ============================= + self.docked_widget_crawl_properties = QDockWidget("Crawl Settings", self) + self.docked_widget_crawl_properties.setMinimumSize(QSize(400, 100)) + + widget_crawl_properties = QWidget() + form_layout_crawl_properties = QFormLayout() + + horizontal_Layout_wordpress = QHBoxLayout() + self.lineedit_wp_user = QLineEdit() + self.lineedit_wp_user.setMaximumWidth(80) + self.lineedit_wp_user.setObjectName("wordpress-user") + self.lineedit_wp_user.textChanged.connect(self.update_windows_title) + horizontal_Layout_wordpress.addWidget(self.lineedit_wp_user) + self.lineedit_wp_api_token = QLineEdit() + self.lineedit_wp_api_token.setObjectName("wordpress-api-token") + self.lineedit_wp_api_token.textChanged.connect(self.update_windows_title) + horizontal_Layout_wordpress.addWidget(QLabel("API Token")) + horizontal_Layout_wordpress.addWidget(self.lineedit_wp_api_token) + form_layout_crawl_properties.addRow( + QLabel("WordPress User"), horizontal_Layout_wordpress + ) + + horizontal_Layout_redirects = QHBoxLayout() + self.combobox_redirects = QComboBox() + self.combobox_redirects.setObjectName("redirects") + self.combobox_redirects.currentTextChanged.connect(self.update_windows_title) + self.combobox_redirects.addItems([item.value for item in list(REDIRECTS)]) + horizontal_Layout_redirects.addWidget(self.combobox_redirects) + form_layout_crawl_properties.addRow( + QLabel("Redirects Source"), horizontal_Layout_redirects + ) + + horizontal_Layout_sitemap = QHBoxLayout() + self.lineedit_sitemap = QLineEdit() + self.lineedit_sitemap.setObjectName("sitemap") + self.lineedit_sitemap.textChanged.connect(self.update_windows_title) + self.toolbutton_output_sitemap = QToolButton() + self.toolbutton_output_sitemap.setIcon( + QIcon(f"{SHARE_FOLDER_PATH}/icons/search.svg") + ) + horizontal_Layout_sitemap.addWidget(self.lineedit_sitemap) + horizontal_Layout_sitemap.addWidget(self.toolbutton_output_sitemap) + self.toolbutton_output_sitemap.clicked.connect(self.get_sitemap_location) + form_layout_crawl_properties.addRow( + QLabel("Sitemap Location"), horizontal_Layout_sitemap + ) + + horizontal_Layout_search_404 = QHBoxLayout() + self.lineedit_search = QLineEdit() + self.lineedit_search.setObjectName("search") + self.lineedit_search.textChanged.connect(self.update_windows_title) + horizontal_Layout_search_404.addWidget(self.lineedit_search) + horizontal_Layout_search_404.addWidget(QLabel("404 Page")) + self.lineedit_404_page = QLineEdit() + self.lineedit_404_page.setObjectName("404-error") + self.lineedit_404_page.textChanged.connect(self.update_windows_title) + horizontal_Layout_search_404.addWidget(self.lineedit_404_page) + form_layout_crawl_properties.addRow( + QLabel("Search Page"), horizontal_Layout_search_404 + ) + + self.lineedit_delay = QLineEdit() + self.lineedit_delay.setObjectName("delay") + self.lineedit_delay.textChanged.connect(self.update_windows_title) + form_layout_crawl_properties.addRow( + QLabel("Delay (Seconds)"), self.lineedit_delay + ) + + form_layout_crawl_properties.addRow(QLabel("Additional Files")) + self.textedit_additional = QTextEdit() + self.textedit_additional.setObjectName("additional") + self.textedit_additional.textChanged.connect(self.update_windows_title) + form_layout_crawl_properties.addRow(self.textedit_additional) + + form_layout_crawl_properties.addRow(QLabel("Exclude Patterns")) + self.textedit_exclude = QTextEdit() + self.textedit_exclude.setObjectName("exclude") + self.textedit_exclude.textChanged.connect(self.update_windows_title) + form_layout_crawl_properties.addRow(self.textedit_exclude) + + vertical_layout_wp_properties = QVBoxLayout() + vertical_layout_wp_properties.addLayout(form_layout_crawl_properties) + widget_crawl_properties.setLayout(vertical_layout_wp_properties) + self.docked_widget_crawl_properties.setWidget(widget_crawl_properties) + self.docked_widget_crawl_properties.setFeatures( + QDockWidget.NoDockWidgetFeatures + ) + self.addDockWidget(Qt.LeftDockWidgetArea, self.docked_widget_crawl_properties) + + self.text_edit_logging = LoggerWidget(self) + self.text_edit_logging.setFormatter( + logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + ) + logging.getLogger().addHandler(self.text_edit_logging) + logging.getLogger().setLevel(logging.INFO) + self.setCentralWidget(self.text_edit_logging.plainTextEdit) + + self.statusBar().showMessage(f"{CONFIGS['APPLICATION_NAME']} is Ready") + self.progressBar = QProgressBar() + self.progressBar.setAlignment(Qt.AlignCenter) + self.progressBar.setFormat("No Brackground Process is running") + self.progressBar.setFixedSize(QSize(300, 25)) + self.progressBar.setValue(0) + self.statusBar().addPermanentWidget(self.progressBar) + + # ALL menus + for current_menu in GUI_SETTINGS["MENUS"]: + parent = self.findChild(QMenu, current_menu["parent"]) + if not parent: + parent = self.menuBar() + + menu = parent.addMenu(current_menu["text"]) + menu.setObjectName(current_menu["name"]) + menu.setEnabled(current_menu["enable"]) + if current_menu["icon"]: + menu.setIcon(QIcon(f"{SHARE_FOLDER_PATH}{current_menu['icon']}")) + + # all toolbars + for current_toolbar in GUI_SETTINGS["TOOLBARS"]: + toolbar = self.addToolBar(current_toolbar["text"]) + toolbar.setObjectName(current_toolbar["name"]) + toolbar.setEnabled(current_toolbar["enable"]) + + # all actions + for current_action in GUI_SETTINGS["ACTIONS"]: + action = QAction( + QIcon(f"{SHARE_FOLDER_PATH}{ current_action['icon']}"), + current_action["text"], + self, + ) + action.setObjectName(current_action["name"]) + action.setShortcut(current_action["shortcut"]) + action.setStatusTip(current_action["tooltip"]) + action.setToolTip(current_action["tooltip"]) + action.setVisible(current_action["visible"]) + action.triggered.connect(eval(current_action["function"])) + action.setCheckable(current_action["setCheckable"]) + + current_menu = ( + self.findChild(QMenu, current_action["menu"]) + if current_action["menu"] + else None + ) + current_toolbar = ( + self.findChild(QToolBar, current_action["toolbar"]) + if current_action["toolbar"] + else None + ) + + if current_menu: + current_menu.addAction(action) + if current_action["seperator"]: + current_menu.addSeparator() + if current_toolbar: + current_toolbar.addAction(action) + + self.setWindowIcon(QIcon(f"{SHARE_FOLDER_PATH}/icons/static-wordpress.svg")) + self.setWindowTitle(f"{CONFIGS['APPLICATION_NAME']} Version - {VERISON}") + self.setMinimumSize(QSize(1366, 768)) + self.statusBar() + logging.info( + "Loaded static-wordpress Successfully. Open/Create a Project to get started" + ) + logging.info("".join(140 * ["-"])) + self.show() + + # decorators + def is_power_user(func): + def inner(self): + if self.findChild(QAction, "action_edit_expert_mode").isChecked(): + return func(self) + + return inner + + def is_new_project(func): + def inner(self): + if self._project.is_open() or self._project.is_new(): + return func(self) + + return inner + + def is_project_open(func): + def inner(self): + if self._project.is_open(): + return func(self) + + return inner + + def has_project_github(func): + def inner(self): + if self._project.has_github(): + return func(self) + + return inner + + @is_new_project + @logging_decorator + def get_output_directory(self): + """""" + output_directory = QFileDialog.getExistingDirectory( + self, + "Select Output Directory", + self.appConfigurations.value("output-directory"), + ) + if output_directory: + self.lineedit_output.setText(output_directory) + self.appConfigurations.setValue("output-directory", output_directory) + + @is_project_open + @logging_decorator + def clean_output_directory(self): + """Clean Output Directory""" + reply = QMessageBox.question( + self, + "Clean Output Folder Content", + f"Existing content in Output folder will be delete?\n {self._project.output}", + QMessageBox.Yes | QMessageBox.No, + ) + if reply == QMessageBox.Yes: + rm_dir_tree(self._project.output) + logging.info( + f"Content of output folder at {self._project.output} are deleted" + ) + + @is_new_project + @logging_decorator + def get_sitemap_location(self): + """ """ + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_thread.started.connect(self._bg_worker.find_sitemap) + self._bg_worker.signalSitemapLocation.connect(self.update_sitemap_location) + self._bg_thread.start() + + def update_sitemap_location(self, sitemap_location): + self._project.sitemap = sitemap_location + logging.info(f"Found Sitemap location: {sitemap_location}") + self.update_properties_widgets() + + @is_new_project + @logging_decorator + def extract_url_from_raw_text(self): + rtp = RawTextWidget( + src_url=self._project.src_url, dest_url=self._project.dst_url + ) + if rtp.exec_(): + raw_text = rtp.textedit_raw_text_with_links.toPlainText() + current_additional_urls = self.textedit_additional.toPlainText().split("\n") + + if raw_text: + new_additional_links = extract_urls_from_raw_text( + raw_text, rtp.lineedit_dest_url.text(), rtp.linedit_src_url.text() + ) + logging.info(f" {len(new_additional_links)} Additional Urls Found") + current_additional_urls += new_additional_links + self.textedit_additional.setText( + "\n".join(set(current_additional_urls)) + ) + + @is_new_project + @logging_decorator + def clear_cache(self): + """Clearing Crawl Cache""" + logging.info(f"Clearing Crawl Cache") + get_remote_content.cache_clear() + + def closeEvent(self, event): + """ """ + reply = QMessageBox.question( + self, + "Exiting static-wordpress", + "Do you really want to exit?.\nAny unsaved changes will be lost!", + QMessageBox.Yes | QMessageBox.No, + ) + + if reply == QMessageBox.Yes: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + del self._bg_thread + del self._bg_worker + super(StaticWordPressGUI, self).closeEvent(event) + + event.accept() + + else: + event.ignore() + + def set_expert_mode(self): + expert_widgets = [ + "action_wordpress_404_page", + "action_wordpress_redirects", + "action_wordpress_robots_txt", + "action_wordpress_search_index", + "action_wordpress_webpages", + ] + + for widget_name in expert_widgets: + self.findChild(QAction, widget_name).setVisible(self.sender().isChecked()) + + self.findChild(QAction, "action_wordpress_additional_files").setVisible( + self.sender().isChecked() and self._project.src_type != SOURCE.ZIP + ) + + def set_debug_mode(self): + if self.sender().isChecked(): + logging.getLogger().setLevel( + logging.INFO & logging.DEBUG & logging.ERROR & logging.WARNING + ) + else: + logging.getLogger().setLevel(logging.INFO) + + def help(self): + """ """ + url = QUrl(CONFIGS["HELP_URL"]) + + if not QDesktopServices.openUrl(url): + QMessageBox.warning(self, "Open Help URL", "Could not open Help URL") + + @logging_decorator + def show_configs(self): + """Interface with System Configurations""" + w = ConfigWidget() + if w.exec_(): + logging.info("Saved/Updated Default Configurations") + + def about(self): + """ """ + msgBox = QMessageBox() + msgBox.setText( + f"Copyright {date.today().year} - SERP Wings" + f"

{CONFIGS['APPLICATION_NAME']} Version - {VERISON}" + "

This work is an opensource project under GNU General Public License v3 or later (GPLv3+)" + f"
More Information at {CONFIGS['ORGANIZATION_NAME']}" + ) + msgBox.setWindowIcon(QIcon(f"{SHARE_FOLDER_PATH}/icons/static-wordpress.svg")) + msgBox.setTextFormat(Qt.RichText) + msgBox.setWindowTitle("About Us") + msgBox.setStandardButtons(QMessageBox.Ok) + msgBox.exec() + + @logging_decorator + def create_project(self): + """Closing current project will automatically start a new project.""" + self.close_project() + self._project.create() + self.update_properties_widgets() + + @logging_decorator + def open_project(self): + """Opening static-wordpress Project File""" + self.close_project() + if not self._project.is_open(): + options = QFileDialog.Options() + project_path, _ = QFileDialog.getOpenFileName( + self, + "Select Static-WordPress Project File", + self.appConfigurations.value("last-project"), + "JSON Files (*.json)", + options=options, + ) + + if project_path: + self._project.open(Path(project_path)) + + if self._project.is_open(): + if not self._project.is_older_version(): + logging.warning( + f"Your Project was saved with an older version : {self._project.version}." + ) + logging.info(f"Open Project {self._project.path} Successfully") + self.appConfigurations.setValue("last-project", project_path) + else: + msgBox = QMessageBox() + msgBox.setText(f"Project cannot be opened." f"Please try again.") + msgBox.setWindowIcon( + QIcon(f"{SHARE_FOLDER_PATH}/icons/static-wordpress.svg") + ) + msgBox.setTextFormat(Qt.RichText) + msgBox.setWindowTitle("Open Project") + msgBox.setStandardButtons(QMessageBox.Ok) + msgBox.exec() + + logging.info( + "No New Project Opened. Unsaved project properties will be lost." + ) + + self.update_windows_title() + self.update_properties_widgets() + + @is_project_open + @logging_decorator + def close_project(self): + """Assign new project and old properties will be lost. + Default is assigned as CLOSED project + """ + reply = QMessageBox.question( + self, + "Close Existing Project", + "Are you sure to close current project and open new one?.\n All existing project properties will be lost!", + QMessageBox.Yes | QMessageBox.No, + ) + if reply == QMessageBox.Yes: + self._project = Project() + self.update_properties_widgets() + verifications = { + "name": None, + "src-url": None, + "wordpress-user": None, + "wordpress-api-token": None, + "sitemap": None, + "github-token": None, + "github-repository": None, + } + self.update_expert_mode_widgets(verifications) + + @is_project_open + @logging_decorator + def save_project(self): + """Saving Current static-wordpress Project""" + if self.lineedit_project_name.text(): + new_project_path = self.appConfigurations.value("last-project") + if self._project.is_new(): + options = QFileDialog.Options() + new_project_path, _ = QFileDialog.getSaveFileName( + self, + "Select StaticWordPress Project File", + self.appConfigurations.value("last-project"), + "JSON Files (*.json)", + options=options, + ) + if new_project_path: + self._project.path = Path(new_project_path) + else: + return + + self.appConfigurations.setValue("last-project", new_project_path) + self._project.name = self.lineedit_project_name.text() + self._project.path = Path(new_project_path) + self._project.src_url = self.lineedit_src_url.text() + self._project.sitemap = self.lineedit_sitemap.text() + self._project.wp_user = self.lineedit_wp_user.text() + self._project.wp_api_token = self.lineedit_wp_api_token.text() + self._project.search = self.lineedit_search.text() + self._project._404 = self.lineedit_404_page.text() + self._project.delay = float(self.lineedit_delay.text()) + self._project.redirects = REDIRECTS[self.combobox_redirects.currentText()] + self._project.src_type = SOURCE[self.combobox_source_type.currentText()] + self._project.user_agent = USER_AGENT[ + self.combobox_user_agent.currentText() + ] + self._project.host = HOST[self.combobox_project_destination.currentText()] + self._project.output = Path(self.lineedit_output.text()) + self._project.dst_url = self.lineedit_dest_url.text() + self._project.gh_token = self.lineedit_gh_token.text() + self._project.gh_repo = self.lineedit_gh_repo.text() + self._project.additional = self.textedit_additional.toPlainText().split( + "\n" + ) + self._project.exclude = self.textedit_exclude.toPlainText().split("\n") + if not self._project.is_older_version(): + logging.warning( + f"Your Project will be saved with new version number : {VERISON}." + ) + self._project.version = VERISON + + # save project + self._project.save() + if self._project.is_saved(): + logging.info(f"Project Saved Successfully at {self._project.path}") + self.update_properties_widgets() + self.update_windows_title() + + def check_project(self) -> None: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalVerification.connect(self.update_expert_mode_widgets) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.verify_project) + self._bg_thread.start() + + @is_project_open + def start_batch_process(self): + """Start Crawling""" + if not self._project.output.exists(): + reply = QMessageBox.question( + self, + f"Output Folder", + f"Following Output Folder doesnt not exit?.\n{self._project.output}\nDo You want to create it now?", + QMessageBox.Yes | QMessageBox.No, + ) + + if reply == QMessageBox.Yes: + os.mkdir(self._project.output) + else: + return + else: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + + if self._project.src_type == SOURCE.ZIP: + if not self._bg_worker._work_flow.verify_simply_static(): + reply = QMessageBox.question( + self, + f"ZIP File Missing", + f"ZIP File not found. Please check your project configurations?", + QMessageBox.Yes, + ) + if reply == QMessageBox.Yes: + return + + self._bg_thread = QThread(parent=self) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.batch_processing) + self._bg_thread.start() + + @is_project_open + def stop_process(self) -> None: + if self._bg_worker.is_running(): + reply = QMessageBox.question( + self, + "Stop Crawling Process", + "Do you really want to Stop Crawling Thrad?", + QMessageBox.Yes | QMessageBox.No, + ) + + if reply == QMessageBox.Yes: + self._bg_worker.stop_calcualations() + self.update_statusbar_widgets("Stoping Processing", 100) + + @is_project_open + def crawl_website(self) -> None: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.pre_processing) + self._bg_thread.start() + + @is_project_open + def crawl_additional_files(self) -> None: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.crawl_additional_files) + self._bg_thread.start() + + @is_project_open + def create_search_index(self) -> None: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.add_search) + self._bg_thread.start() + + @is_project_open + def create_404_page(self) -> None: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.add_404_page) + self._bg_thread.start() + + @is_project_open + def create_redirects(self) -> None: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.add_redirects) + self._bg_thread.start() + + @is_project_open + def create_robots_txt(self) -> None: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.add_robots_txt) + self._bg_thread.start() + + @is_project_open + def create_github_repositoy(self) -> None: + """""" + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.create_github_repositoy) + self._bg_thread.start() + + @is_project_open + def delete_github_repository(self) -> None: + """""" + reply = QMessageBox.question( + self, + "Deleting Repository on GitHub", + f"Do you really want to delete {self._project.gh_repo} on GitHub?\nThis deletion is not reversible.", + QMessageBox.Yes | QMessageBox.No, + ) + + if reply == QMessageBox.Yes: + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.delete_github_repositoy) + self._bg_thread.start() + + @is_project_open + def initialize_repository(self) -> None: + """""" + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.init_git_repositoy) + self._bg_thread.start() + + @is_project_open + def commit_repository(self) -> None: + """""" + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.commit_git_repositoy) + self._bg_thread.start() + + @is_project_open + def publish_repository(self) -> None: + """ """ + if self._bg_thread.isRunning(): + self._bg_thread.quit() + + self._bg_thread = QThread(parent=self) + self._bg_worker = WorkflowGUI() + self._bg_worker.set_project(project_=self._project) + self._bg_worker.moveToThread(self._bg_thread) + self._bg_thread.finished.connect(self._bg_worker.deleteLater) + self._bg_worker.signalProgress.connect(self.update_statusbar_widgets) + self._bg_thread.started.connect(self._bg_worker.publish_github_repositoy) + self._bg_thread.start() + + def update_statusbar_widgets(self, message_, percent_) -> None: + if percent_ >= 0: + self.progressBar.setValue(percent_) + self.statusBar().showMessage(message_) + else: + self.progressBar.setFormat(message_) + + if percent_ >= 100: + self.progressBar.setFormat(message_) + + def update_properties_widgets(self) -> None: + self.lineedit_project_name.setText(self._project.name) + self.lineedit_src_url.setText(self._project.src_url) + self.lineedit_sitemap.setText(self._project.sitemap) + self.lineedit_search.setText(self._project.search) + self.lineedit_404_page.setText(self._project._404) + self.lineedit_delay.setText(f"{self._project.delay}") + self.combobox_source_type.setCurrentText(self._project.src_type.value) + self.combobox_user_agent.setCurrentText(self._project.user_agent.value) + self.lineedit_output.setText(str(self._project.output)) + self.lineedit_dest_url.setText(self._project.dst_url) + self.lineedit_wp_user.setText(self._project.wp_user) + self.lineedit_wp_api_token.setText(self._project.wp_api_token) + self.lineedit_gh_token.setText(self._project.gh_token) + self.lineedit_gh_repo.setText(self._project.gh_repo) + self.textedit_additional.setText("\n".join(self._project.additional)) + self.textedit_exclude.setText("\n".join(self._project.exclude)) + self.combobox_redirects.setCurrentText(self._project.redirects.value) + self.combobox_redirects.setEnabled(True) + + self.findChild(QMenu, "menu_github").setEnabled(self._project.has_github()) + self.findChild(QMenu, "menu_wordpress").setEnabled( + self._project.has_wordpress() + ) + self.findChild(QToolBar, "toolbar_github").setEnabled( + self._project.has_github() + ) + self.findChild(QToolBar, "toolbar_wordpres").setEnabled( + self._project.has_wordpress() + ) + + # update menu items + if self._project.src_type == SOURCE.ZIP: + self.findChild(QAction, "action_wordpress_webpages").setText( + "&Download Zip File" + ) + self.findChild(QAction, "action_wordpress_additional_files").setVisible( + False + ) + else: + self.findChild(QAction, "action_wordpress_webpages").setText( + "&Crawl Webpages" + ) + self.findChild(QAction, "action_wordpress_additional_files").setVisible( + self.findChild(QAction, "action_edit_expert_mode").isChecked() + ) + + def update_expert_mode_widgets(self, verifications) -> None: + for key, value in verifications.items(): + if value is not None: + bg_color = ( + CONFIGS["COLOR"]["SUCCESS"] if value else CONFIGS["COLOR"]["ERROR"] + ) + else: + bg_color = value + + self.findChild(QLineEdit, key).setStyleSheet( + f"background-color: {bg_color}" + ) + + def update_windows_title(self) -> None: + sender = self.sender() + if ( + type(sender) in [QLineEdit, QComboBox, QTextEdit] + and self._project.is_open() + ): + gui_value = sender.property("text") + + if type(sender) == QLineEdit: + gui_value = sender.property("text") + if sender.objectName() == "output": + gui_value = Path(sender.property("text")) + elif sender.objectName() == "delay": + try: + gui_value = float(sender.property("text")) + except: # mostly value error e.g. 0.^ as input + pass + + elif type(sender) == QComboBox: + gui_value = ENUMS_MAP[sender.objectName()][ + sender.property("currentText") + ] + elif type(sender) == QTextEdit: + gui_value = [url for url in sender.toPlainText().split("\n") if url] + + project_value = self._project.get(sender.objectName()) + project_functions_dict = { + "github-repository": self._project.gh_repo, + "github-token": self._project.gh_token, + "wordpress-user": self._project.wp_user, + "wordpress-api-token": self._project.wp_api_token, + "src-url": self._project.src_url, + "source": self._project.src_type, + "output": self._project.output, + "dst-url": self._project.dst_url, + "404-error": self._project._404, + } + + if sender.objectName() in project_functions_dict: + project_value = project_functions_dict[sender.objectName()] + + if gui_value != project_value and not self._project.is_new(): + self._project.status = PROJECT.UPDATE + + status_string = ( + f"{'' if self._project.is_saved() else '*'} {self._project.name}" + ) + new_window_title = ( + f"{status_string} - {CONFIGS['APPLICATION_NAME']} Version - {VERISON}" + ) + if self._project.status == PROJECT.NOT_FOUND: + new_window_title = f"{CONFIGS['APPLICATION_NAME']} Version - {VERISON}" + + self.setWindowTitle(new_window_title) + + +def main(): + app = QApplication(sys.argv) + wind = StaticWordPressGUI() + sys.exit(app.exec_()) + + +if __name__ == "__main__": + pass diff --git a/src/staticwordpress/gui/rawtext.py b/src/staticwordpress/gui/rawtext.py new file mode 100644 index 0000000..d766a2c --- /dev/null +++ b/src/staticwordpress/gui/rawtext.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\gui\rawtext.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from PyQt5.QtWidgets import ( + QDialog, + QDialogButtonBox, + QVBoxLayout, + QFormLayout, + QLabel, + QLineEdit, + QTextEdit, +) +from PyQt5.QtGui import QIcon +from PyQt5.QtCore import QSettings + + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from ..core.constants import CONFIGS, SHARE_FOLDER_PATH + + +class RawTextWidget(QDialog): + def __init__(self, src_url: str, dest_url: str): + super(RawTextWidget, self).__init__() + self.appConfigurations = QSettings( + CONFIGS["APPLICATION_NAME"], CONFIGS["APPLICATION_NAME"] + ) + + self.textedit_raw_text_with_links = QTextEdit() + self.linedit_src_url = QLineEdit() + self.linedit_src_url.setText(src_url) + self.lineedit_dest_url = QLineEdit() + self.lineedit_dest_url.setText(dest_url) + + form_layout_raw_text = QFormLayout() + form_layout_raw_text.addRow(QLabel("Raw Text With Links")) + form_layout_raw_text.addRow(self.textedit_raw_text_with_links) + form_layout_raw_text.addRow(QLabel("Search Url:"), self.lineedit_dest_url) + form_layout_raw_text.addRow(QLabel("Replace Url:"), self.linedit_src_url) + + dialog_button_box = QDialogButtonBox( + QDialogButtonBox.Ok | QDialogButtonBox.Cancel + ) + dialog_button_box.accepted.connect(self.accept) + dialog_button_box.rejected.connect(self.reject) + + vertical_layout_main = QVBoxLayout() + vertical_layout_main.addLayout(form_layout_raw_text) + vertical_layout_main.addWidget(dialog_button_box) + self.setLayout(vertical_layout_main) + self.setMinimumWidth(400) + self.setWindowTitle("Processing Raw Text") + self.setWindowIcon(QIcon(f"{SHARE_FOLDER_PATH}/icons/static-wordpress.svg")) diff --git a/src/staticwordpress/gui/utils.py b/src/staticwordpress/gui/utils.py new file mode 100644 index 0000000..e51f936 --- /dev/null +++ b/src/staticwordpress/gui/utils.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\gui\utils.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import logging +import json + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# IMPLEMENATIONS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +from ..core.constants import SHARE_FOLDER_PATH + +GUI_JSON_PATH = SHARE_FOLDER_PATH / "gui.json" +with GUI_JSON_PATH.open("r") as f: + GUI_SETTINGS = json.load(f) + + +def logging_decorator(function): + def add_logs(cls): + if function.__doc__: + logging.info(function.__doc__.split("\n")[0]) + + logging.info(f"Start Function: {function.__name__}") + try: + function(cls) + logging.info(f"Stop Function: {function.__name__}") + except: + logging.error(f"Failed Function: {function.__name__}") + + logging.info("".join(140 * ["-"])) + + return add_logs + + +def progress_decorator(message, percent=10): + """ """ + + def decorator(function): + def wrapper(cls): + logging.info(f"{message}") + cls.signalProgress.emit(message, -1) + cls.signalProgress.emit(message, percent) + result = function(cls) + return result + + return wrapper + + return decorator diff --git a/src/staticwordpress/gui/workflow.py b/src/staticwordpress/gui/workflow.py new file mode 100644 index 0000000..bc5a550 --- /dev/null +++ b/src/staticwordpress/gui/workflow.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + src\staticwordpress\gui\workflow.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# 3rd PARTY LIBRARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from PyQt5.QtCore import QObject, pyqtSignal + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from ..core.project import Project +from ..core.constants import SOURCE +from ..core.workflow import Workflow +from .utils import logging_decorator, progress_decorator + + +class WorkflowGUI(QObject): + signalSitemapLocation = pyqtSignal(str) + signalProgress = pyqtSignal(str, int) + signalVerification = pyqtSignal(dict) + + _work_flow = Workflow() + + def set_project(self, project_: Project) -> None: + if project_.is_open(): + self._work_flow.set_project(project_=project_) + + @progress_decorator("Verifying Project Settings", 100) + @logging_decorator + def verify_project(self): + self.signalVerification.emit( + { + "name": self._work_flow.verify_project_name(), + "src-url": self._work_flow.verify_src_url(), + "wordpress-user": self._work_flow.verify_wp_user(), + "wordpress-api-token": self._work_flow.verify_wp_user(), + "sitemap": self._work_flow.verify_sitemap(), + "github-token": self._work_flow.verify_github_token(), + "github-repository": self._work_flow.verify_github_repo(), + } + ) + + @logging_decorator + def stop_calcualations(self): + self._work_flow.stop_calculations() + + def is_running(self): + return self._work_flow._keep_running + + def batch_processing(self): + self._work_flow._keep_running = True + if self._work_flow._project.src_type == SOURCE.CRAWL: + self.crawl_sitemap() + self.crawl_additional_files() + + self.pre_processing() + self.add_404_page() + self.add_robots_txt() + self.add_redirects() + self.add_search() + + @progress_decorator("Pre-Processing", 100) + @logging_decorator + def pre_processing(self): + if self._work_flow._project.src_type == SOURCE.ZIP: + self.download_zip_file() + self.setup_zip_folders() + else: + self.crawl_url(loc_=self._work_flow._project.src_url) + + @progress_decorator("Downloading Zip File", 100) + @logging_decorator + def download_zip_file(self) -> None: + self._work_flow.download_zip_file() + + @progress_decorator("Setting Up Directories", 100) + def setup_zip_folders(self) -> None: + self._work_flow.setup_zip_folders() + + @progress_decorator("Saving 404 Page", 100) + @logging_decorator + def add_404_page(self) -> None: + self._work_flow.add_404_page() + + @progress_decorator("Copying robots.txt", 100) + @logging_decorator + def add_robots_txt(self) -> None: + self._work_flow.add_robots_txt() + + @progress_decorator("Writing Redirects File", 100) + @logging_decorator + def add_redirects(self) -> None: + self._work_flow.add_redirects() + + @progress_decorator("Generating Search Index", 100) + @logging_decorator + def add_search(self) -> None: + self._work_flow.add_search() + + @progress_decorator("Searching Sitemap", 100) + @logging_decorator + def find_sitemap(self) -> None: + self._work_flow.find_sitemap() + self.signalSitemapLocation.emit(self._work_flow.sitemap) + + @progress_decorator("Crawling Sitemap", 100) + @logging_decorator + def crawl_sitemap(self) -> None: + self._work_flow.crawl_sitemap() + + def crawl_url(self, loc_): + self._work_flow.clear() + self._work_flow.crawl_url(loc_=loc_) + + @progress_decorator("Crawling Additional Pages", 100) + @logging_decorator + def crawl_additional_files(self) -> None: + for additiona_url in self._work_flow._project.additional: + self.crawl_url(loc_=additiona_url) + + @progress_decorator("Creating Website on GitHub", 100) + @logging_decorator + def create_github_repositoy(self): + self._work_flow.create_github_repositoy() + + @progress_decorator("Deleting from GitHub", 100) + @logging_decorator + def delete_github_repositoy(self): + self._work_flow.delete_github_repositoy() + + @progress_decorator("Initalizing Website", 100) + @logging_decorator + def init_git_repositoy(self): + self._work_flow.init_git_repositoy() + + @progress_decorator("Commiting Website to Repo", 100) + @logging_decorator + def commit_git_repositoy(self): + self._work_flow.commit_git_repositoy() + + @progress_decorator("Pushing Website to GitHub", 100) + @logging_decorator + def publish_github_repositoy(self): + self._work_flow.publish_github_repositoy() diff --git a/src/staticwordpress/share/config.json b/src/staticwordpress/share/config.json new file mode 100644 index 0000000..98e2b00 --- /dev/null +++ b/src/staticwordpress/share/config.json @@ -0,0 +1,713 @@ +{ + "LANGUAGE": "en_US", + "APPLICATION_NAME": "Static WordPress", + "ORGANIZATION_NAME": "SERP Wings", + "ORGANIZATION_DOMAIN": "serpwings.com", + "HELP_URL": "https://serpwings.com/staticwordpress/", + "REPO_URL": "https://github.com/serpwings/staticwordpress/", + "SCHEMES": [ + "http", + "https", + "ftp", + "sftp" + ], + "DEFAULT_SCHEME": "http", + "DEFAULT_USER_AGENT": "CHROME", + "CLEAN": { + "URL": true, + "CHARS": "()[]" + }, + "HEADER": { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET", + "Access-Control-Allow-Headers": "Content-Type", + "Access-Control-Max-Age": "3600", + "FIREFOX": { + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0" + }, + "CHROME": { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" + }, + "CUSTOM": { + "User-Agent": "ua: staticwordpress.com" + } + }, + "LUNR": { + "src": "https://cdnjs.cloudflare.com/ajax/libs/lunr.js/2.3.9/lunr.min.js", + "integrity": "sha512-4xUl/d6D6THrAnXAwGajXkoWaeMNwEKK4iNfq5DotEbLPAfk6FSxSP3ydNxqDgCw1c/0Z1Jg6L8h2j+++9BZmg==", + "crossorigin": "anonymous", + "referrerpolicy": "no-referrer" + }, + "XSL_ATTR_LIST": { + "xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9", + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xmlns:image": "http://www.google.com/schemas/sitemap-image/1.1" + }, + "COLOR": { + "SUCCESS": "#005500", + "WARNING": "#ffff7f", + "ERROR": "#ff557f" + }, + "EXCLUDE": [ + ".php", + "wp-json", + "wp-admin", + "feed" + ], + "SIMPLYSTATIC": { + "API": "/wp-json/simplystatic/v1/settings", + "FOLDER": "/wp-content/uploads/simply-static/temp-files/" + }, + "REDIRECTS": { + "REDIRECTION": { + "API": "/wp-json/redirection/v1/redirect" + }, + "NONE": { + "API": "" + }, + "DESTINATION": { + "NETLIFY": "netlify.toml", + "LOCALHOST": "_redirects", + "CLOUDFLARE": "_redirects" + } + }, + "SEARCH": { + "INDEX": { + "src": "search.js" + }, + "INCLUDE": { + "TITLE": true, + "CONTENT": true, + "URL": true, + "IMAGE": false + }, + "HTML_TAGS": [ + "h1", + "h2", + "h3" + ] + }, + "ROBOTS_TXT": [ + "User-agent: *", + "Disallow: /wp-admin/", + "Allow: /wp-admin/admin-ajax.php " + ], + "SITEMAP": { + "AUTO": true, + "SEARCH_PATHS": [ + "wp-sitemap.xml", + "sitemap_index.xml", + "sitemap.xml", + "sitemapindex.xml", + "sitemap-index.xml", + "sitemap1.xml", + "sitemap.php", + "sitemap/", + "sitemaps/", + "sitemas.xml", + "sitemap.txt", + "sitemap.xml.gz", + "post-sitemap.xml", + "page-sitemap.xml", + "news-sitemap.xml" + ] + }, + "FORMATS": { + "IMAGE": [ + "001", + "1SC", + "2BP", + "360", + "73I", + "8CA", + "8CI", + "8PBS", + "8XI", + "9.PNG", + "AB3", + "ABM", + "ACCOUNTPICTURE-MS", + "ACORN", + "AR", + "ADC", + "AFPHOTO", + "AFX", + "AGIF", + "AGP", + "AIC", + "AIS", + "APD", + "APNG", + "APS", + "APX", + "ARR", + "ART", + "ART", + "ASE", + "ASEPRITE", + "AVATAR", + "AVB", + "AVIF", + "AVIFS", + "AWD", + "AWD", + "BIF", + "BLKRT", + "BM2", + "BMC", + "BMF", + "BMP", + "BMQ", + "BMX", + "BMZ", + "BPG", + "BRN", + "BRT", + "BS", + "BSS", + "BTI", + "BW", + "C4", + "CAL", + "CALS", + "CAM", + "CAN", + "CAN", + "CD5", + "CDC", + "CDG", + "CE", + "CID", + "CIMG", + "CIN", + "CIT", + "CLIP", + "CMR", + "COLZ", + "CPBITMAP", + "CPC", + "CPD", + "CPG", + "CPS", + "CPT", + "CPX", + "CSF", + "CT", + "CUT", + "DC2", + "DC6", + "DCM", + "DCX", + "DDB", + "DDS", + "DDT", + "DGT", + "DIB", + "DIC", + "DICOM", + "DJV", + "DJVU", + "DM3", + "DM4", + "DMI", + "DPX", + "DRP", + "DRZ", + "DT2", + "DTW", + "DVL", + "ECW", + "EPP", + "EXR", + "FAC", + "FACE", + "FAX", + "FBM", + "FF", + "FIL", + "FITS", + "FLIF", + "FPOS", + "FPPX", + "FPX", + "FRM", + "FSTHUMB", + "FSYMBOLS-ART", + "G3F", + "G3N", + "GBR", + "GCDP", + "GFB", + "GFIE", + "GGR", + "GIF", + "GIH", + "GIM", + "GMBCK", + "GMSPR", + "GP4", + "GPD", + "GRO", + "GVRS", + "HDP", + "HDR", + "HDRP", + "HEIC", + "HEIF", + "HF", + "HIF", + "HPI", + "HR", + "I3D", + "IC1", + "IC2", + "IC3", + "ICA", + "ICN", + "ICON", + "ICO", + "ICPR", + "ILBM", + "INFO", + "INK", + "INSP", + "INT", + "IPHOTOPROJECT", + "IPICK", + "IPV", + "IPX", + "ITC2", + "ITHMB", + "IVR", + "IVUE", + "IWI", + "J", + "J2C", + "J2K", + "JAS", + "JB2", + "JBF", + "JBG", + "JBIG", + "JBIG2", + "JBR", + "JFI", + "JFIF", + "JIA", + "JIF", + "JIFF", + "JLS", + "JNG", + "JP2", + "JPC", + "JPD", + "JPE", + "JPEG", + "JPF", + "JPG", + "JPG2", + "JPG_LARGE", + "JPS", + "JPX", + "JTF", + "JWL", + "JXL", + "JXR", + "KDI", + "KFX", + "KIC", + "KODAK", + "KPG", + "KRA", + "KRA~", + "KTX", + "KTX2", + "LB", + "LBM", + "LDOC", + "LIF", + "LINEA", + "LIP", + "LJP", + "LMNR", + "LRPREVIEW", + "LZP", + "MAC", + "MAT", + "MAX", + "MBM", + "MBM", + "MCS", + "MDP", + "MDP", + "MIC", + "MIFF", + "MIP", + "MIPMAPS", + "MIX", + "MNG", + "MNR", + "MONOPIC", + "MONOSNIPPET", + "MPF", + "MPO", + "MRB", + "MRXS", + "MSK", + "MSP", + "MXI", + "MYL", + "NCD", + "NCR", + "NCT", + "NDPI", + "NEO", + "NLM", + "NOL", + "NPSD", + "NWM", + "OC3", + "OC4", + "OC5", + "OCI", + "ODI", + "OE6", + "OIR", + "OMF", + "OPLC", + "ORA", + "OTA", + "OTB", + "OTI", + "OZB", + "OZJ", + "OZT", + "PAC", + "PAL", + "PALM", + "PAM", + "PANO", + "PAP", + "PAT", + "PBM", + "PBS", + "PC1", + "PC2", + "PC3", + "PCD", + "PCX", + "PDD", + "PDN", + "PE4", + "PE4", + "PFI", + "PFR", + "PGF", + "PGM", + "PI1", + "PI2", + "PIC", + "PIC", + "PICNC", + "PICTCLIPPING", + "PISKEL", + "PIX", + "PIXADEX", + "PIXELA", + "PJP", + "PJPEG", + "PJPG", + "PLP", + "PM", + "PMG", + "PNC", + "PNG", + "PNI", + "PNM", + "PNS", + "PNT", + "POP", + "POV", + "POV", + "PP4", + "PP5", + "PPF", + "PPM", + "PPP", + "PROCREATE", + "PRW", + "PSB", + "PSD", + "PSDB", + "PSDC", + "PSDX", + "PSE", + "PSF", + "PSP", + "PSP", + "PSPBRUSH", + "PSPFRAME", + "PSPIMAGE", + "PSPTUBE", + "PSXPRJ", + "PTEX", + "PTG", + "PTK", + "PTS", + "PTX", + "PTX", + "PVR", + "PWP", + "PX", + "PXD", + "PXD", + "PXICON", + "PXM", + "PXR", + "PXZ", + "PYXEL", + "PZA", + "PZP", + "PZS", + "QIF", + "QMG", + "QOI", + "QPTIFF", + "QTI", + "QTIF", + "RAS", + "RCL", + "RCU", + "RGB", + "RGB", + "RGBA", + "RGF", + "RIC", + "RIF", + "RIFF", + "RLE", + "RLI", + "RPF", + "RRI", + "RSB", + "RSR", + "RTL", + "RVG", + "S2MV", + "SAI", + "SAI2", + "SBP", + "SCN", + "SCN", + "SCN", + "SCT", + "SDR", + "SEP", + "SFC", + "SFF", + "SFW", + "SGD", + "SGI", + "SHG", + "SID", + "SID", + "SIG", + "SIG", + "SIM", + "SIX", + "SKITCH", + "SKM", + "SKTZ", + "SKY", + "SKYPEEMOTICONSET", + "SLD", + "SMP", + "SNAG", + "SNAGPROJ", + "SNAGX", + "SOB", + "SPA", + "SPE", + "SPH", + "SPJ", + "SPP", + "SPR", + "SPRITE", + "SPRITE2", + "SPRITE3", + "SRF", + "STEX", + "SUMO", + "SUN", + "SUP", + "SVA", + "SVS", + "SVSLIDE", + "T2B", + "T2K", + "TARGA", + "TBN", + "TEX", + "TEXTURE", + "TFC", + "TG4", + "TGA", + "THM", + "THM", + "THUMB", + "TIF", + "TIF", + "TIFF", + "TJP", + "TLA", + "TM2", + "TN", + "TPF", + "TPS", + "TRIF", + "TSR", + "TUB", + "U", + "UFO", + "UGA", + "UGOIRA", + "UPF", + "URT", + "USERTILE-MS", + "V", + "VDA", + "VDOC", + "VIC", + "VICAR", + "VIFF", + "VMU", + "VNA", + "VPE", + "VRIMG", + "VRPHOTO", + "VSS", + "VST", + "WB0", + "WB1", + "WB2", + "WBC", + "WBD", + "WBM", + "WBMP", + "WBP", + "WBZ", + "WDP", + "WEBP", + "WI", + "WIC", + "WMP", + "WPB", + "WPE", + "WVL", + "XBM", + "XCF", + "XFACE", + "XPM", + "XWD", + "Y", + "YSP", + "YUV", + "ZIF", + "ZIF", + "ZVI", + "SVG" + ], + "FONTS": [ + "ABF", + "ACFM", + "AFM", + "AMFM", + "BDF", + "BF", + "BMFC", + "CHA", + "CHR", + "COMPOSITEFONT", + "DFONT", + "EOT", + "ETX", + "EUF", + "F3F", + "FEA", + "FFIL", + "FNT", + "FNT", + "FON", + "FOT", + "GDR", + "GF", + "GLIF", + "GXF", + "JFPROJ", + "LWFN", + "MCF", + "MF", + "MXF", + "NFTR", + "ODTTF", + "OTF", + "PCF", + "PF2", + "PFA", + "PFB", + "PFM", + "PFR", + "PFT", + "PK", + "PMT", + "SFD", + "SFP", + "SFT", + "SUIT", + "T65", + "TFM", + "TTC", + "TTE", + "TTF", + "TXF", + "UFO", + "VFB", + "VLW", + "VNF", + "WOFF", + "WOFF2", + "XFN", + "XFT", + "YTF" + ], + "JS": [ + "JS" + ], + "CSS": [ + "CSS" + ], + "PDF": [ + "PDF" + ], + "TXT": [ + "TXT" + ], + "XML": [ + "XML", + "XSL" + ], + "ZIP": [ + "ZIP" + ], + "HTML": [ + "HTML", + "HTM" + ], + "HOME": [ + "/" + ], + "JSON": [ + "JSON" + ] + } +} \ No newline at end of file diff --git a/src/staticwordpress/share/gui.json b/src/staticwordpress/share/gui.json new file mode 100644 index 0000000..1645a55 --- /dev/null +++ b/src/staticwordpress/share/gui.json @@ -0,0 +1,428 @@ +{ + "MENUS": [ + { + "name": "menu_file", + "text": "&File", + "enable": true, + "icon": "", + "parent": "" + }, + { + "name": "menu_edit", + "text": "&Edit", + "enable": true, + "icon": "", + "parent": "" + }, + { + "name": "menu_wordpress", + "text": "&WordPress", + "enable": false, + "icon": "", + "parent": "" + }, + { + "name": "menu_github", + "text": "&GitHub", + "enable": false, + "icon": "", + "parent": "" + }, + { + "name": "menu_github_repository", + "text": "&GitHub", + "enable": true, + "icon": "/icons/github.svg", + "parent": "menu_github" + }, + { + "name": "menu_tools", + "text": "&Tools", + "enable": true, + "icon": "", + "parent": "" + }, + { + "name": "menu_help", + "text": "&Help", + "enable": true, + "icon": "", + "parent": "" + } + ], + "TOOLBARS": [ + { + "name": "toolbar_project", + "text": "Project", + "enable": true + }, + { + "name": "toolbar_wordpres", + "text": "WordPress", + "enable": true + }, + { + "name": "toolbar_github", + "text": "GitHub", + "enable": true + }, + { + "name": "toolbar_edit", + "text": "Edit", + "enable": true + } + ], + "ACTIONS": [ + { + "icon": "/icons/file-outline.svg", + "name": "action_file_project_new", + "visible": true, + "text": "&New Project", + "shortcut": "Ctrl+N", + "tooltip": "New Project (Ctrl+N)", + "function": "self.create_project", + "setCheckable": false, + "menu": "menu_file", + "seperator": false, + "toolbar": "toolbar_project" + }, + { + "icon": "/icons/folder-outline.svg", + "name": "action_file_project_open", + "visible": true, + "text": "&Open Project", + "shortcut": "Ctrl+O", + "tooltip": "Open Project (Ctrl+O)", + "function": "self.open_project", + "setCheckable": false, + "menu": "menu_file", + "seperator": false, + "toolbar": "toolbar_project" + }, + { + "icon": "/icons/content-save-outline.svg", + "name": "action_file_project_save", + "visible": true, + "text": "&Save Project", + "shortcut": "Ctrl+S", + "tooltip": "Save Project (Ctrl+S)", + "function": "self.save_project", + "setCheckable": false, + "menu": "menu_file", + "seperator": false, + "toolbar": "toolbar_project" + }, + { + "icon": "/icons/file-document-remove-outline.svg", + "name": "action_file_project_close", + "visible": true, + "text": "&Close Project", + "shortcut": "Ctrl+X", + "tooltip": "Close Project (Ctrl+X)", + "function": "self.close_project", + "setCheckable": false, + "menu": "menu_file", + "seperator": true, + "toolbar": "toolbar_project" + }, + { + "icon": "/icons/exit-to-app.svg", + "name": "action_file_exit", + "visible": true, + "text": "&Exit", + "shortcut": "Ctrl+Q", + "tooltip": "Exit (Ctrl+Q)", + "function": "self.close", + "setCheckable": false, + "menu": "menu_file", + "seperator": false, + "toolbar": "" + }, + { + "icon": "/icons/bug-outline.svg", + "name": "action_edit_debug", + "visible": true, + "text": "&Debug", + "shortcut": "F5", + "tooltip": "Debug (F5)", + "function": "self.set_debug_mode", + "setCheckable": true, + "menu": "menu_edit", + "seperator": false, + "toolbar": "toolbar_edit" + }, + { + "icon": "/icons/person.svg", + "name": "action_edit_expert_mode", + "visible": true, + "text": "&Expert Mode", + "shortcut": "F8", + "tooltip": "Expert Mode (F8)", + "function": "self.set_expert_mode", + "setCheckable": true, + "menu": "menu_edit", + "seperator": false, + "toolbar": "toolbar_edit" + }, + { + "icon": "/icons/configs.svg", + "name": "action_file_project_close", + "visible": true, + "text": "&Configurations", + "shortcut": "F12", + "tooltip": "Show Configurations (F12)", + "function": "self.show_configs", + "setCheckable": false, + "menu": "menu_edit", + "seperator": true, + "toolbar": "toolbar_edit" + }, + { + "icon": "/icons/play.svg", + "name": "action_wordpress_batch_processing", + "visible": true, + "text": "Start &Batch Process", + "shortcut": "Ctrl+F6", + "tooltip": "Start Batch Process (Ctrl+F6)", + "function": "self.start_batch_process", + "setCheckable": false, + "menu": "menu_wordpress", + "seperator": false, + "toolbar": "toolbar_wordpres" + }, + { + "icon": "/icons/stop.svg", + "name": "action_wordpress_stop_processing", + "visible": true, + "text": "&Stop Process", + "shortcut": "Ctrl+F7", + "tooltip": "Stop Current Process (Ctrl+F7)", + "seperator": false, + "function": "self.stop_process", + "setCheckable": false, + "menu": "menu_wordpress", + "toolbar": "toolbar_wordpres" + }, + { + "icon": "/icons/crawl_website.svg", + "name": "action_wordpress_webpages", + "visible": false, + "text": "&Crawl Webpages", + "shortcut": "Ctrl+1 (Ctrl+1)", + "tooltip": "Crawl Webpages", + "seperator": false, + "function": "self.crawl_website", + "setCheckable": false, + "menu": "menu_wordpress", + "toolbar": "" + }, + { + "icon": "/icons/additionals.svg", + "name": "action_wordpress_additional_files", + "visible": false, + "text": "Crawl &Additional Files", + "shortcut": "Ctrl+2", + "tooltip": "Crawl Additional Files (Ctrl+2)", + "seperator": false, + "function": "self.crawl_additional_files", + "setCheckable": false, + "menu": "menu_wordpress", + "toolbar": "" + }, + { + "icon": "/icons/search.svg", + "name": "action_wordpress_search_index", + "visible": false, + "text": "Prepare &Search Index", + "shortcut": "Ctrl+3", + "tooltip": "Prepare Search Index (Ctrl+3)", + "seperator": false, + "function": "self.create_search_index", + "setCheckable": false, + "menu": "menu_wordpress", + "toolbar": "" + }, + { + "icon": "/icons/error.svg", + "name": "action_wordpress_404_page", + "visible": false, + "text": "Create &404 Page", + "shortcut": "Ctrl+4", + "tooltip": "Create 404 Page (Ctrl+4)", + "seperator": false, + "function": "self.create_404_page", + "setCheckable": false, + "menu": "menu_wordpress", + "toolbar": "" + }, + { + "icon": "/icons/redirects.svg", + "name": "action_wordpress_redirects", + "visible": false, + "text": "Create &Redirects", + "shortcut": "Ctrl+5", + "tooltip": "Create Redirects (Ctrl+5)", + "seperator": false, + "function": "self.create_redirects", + "setCheckable": false, + "menu": "menu_wordpress", + "toolbar": "" + }, + { + "icon": "/icons/robots_txt.svg", + "name": "action_wordpress_robots_txt", + "visible": false, + "text": "Create Robots.&txt", + "shortcut": "Ctrl+6", + "tooltip": "Copy Robots.txt File (Ctrl+6)", + "seperator": false, + "function": "self.create_robots_txt", + "setCheckable": false, + "menu": "menu_wordpress", + "toolbar": "" + }, + { + "icon": "/icons/pencil-outline.svg", + "name": "action_github_create_repositoy", + "visible": true, + "text": "&Create on GitHub", + "shortcut": "Ctrl+F9", + "tooltip": "Create GitHub Repository (Ctrl+F9)", + "function": "self.create_github_repositoy", + "setCheckable": false, + "menu": "menu_github_repository", + "seperator": false, + "toolbar": "" + }, + { + "icon": "/icons/delete-repo.svg", + "name": "action_github_delete_repositoy", + "visible": true, + "text": "&Delete from GitHub", + "shortcut": "Ctrl+Shift+F9", + "tooltip": "Delete Repository on GitHub (Ctrl+Shit+F9)", + "function": "self.delete_github_repository", + "setCheckable": false, + "menu": "menu_github_repository", + "seperator": false, + "toolbar": "" + }, + { + "icon": "/icons/folder-git.svg", + "name": "action_github_initialize_repositoy", + "visible": true, + "text": "&Initialize Website", + "shortcut": "Ctrl+F10", + "tooltip": "Initialize Website (Ctrl+F10)", + "function": "self.initialize_repository", + "setCheckable": false, + "menu": "menu_github", + "seperator": false, + "toolbar": "toolbar_github" + }, + { + "icon": "/icons/playlist-plus.svg", + "name": "action_github_commit_repositoy", + "visible": true, + "text": "&Update Content", + "shortcut": "Ctrl+F11", + "tooltip": "Update Content (Ctrl+F11)", + "function": "self.commit_repository", + "setCheckable": false, + "menu": "menu_github", + "seperator": false, + "toolbar": "toolbar_github" + }, + { + "icon": "/icons/cloud-upload-outline.svg", + "name": "action_github_publish_repositoy", + "visible": true, + "text": "&Publish Website", + "shortcut": "Ctrl+F12", + "tooltip": "Publish Website (Ctrl+F12)", + "function": "self.publish_repository", + "setCheckable": false, + "menu": "menu_github", + "seperator": false, + "toolbar": "toolbar_github" + }, + { + "icon": "/icons/web-remove.svg", + "name": "action_utilities_clear_cache", + "visible": true, + "text": "C&lear Cache", + "shortcut": "Ctrl+F2", + "tooltip": "Clear Cache (Ctrl+F2)", + "seperator": false, + "function": "self.clear_cache", + "setCheckable": false, + "menu": "menu_tools", + "toolbar": "" + }, + { + "icon": "/icons/check_project.svg", + "name": "action_file_check_project", + "visible": true, + "text": "&Check Project", + "shortcut": "Ctrl+F3", + "tooltip": "Check Project (Ctrl+F3)", + "function": "self.check_project", + "setCheckable": false, + "menu": "menu_tools", + "seperator": false, + "toolbar": false + }, + { + "icon": "/icons/text-recognition.svg", + "name": "action_utilities_extract_url_from_raw_text", + "visible": true, + "text": "&Extract Urls From Text", + "shortcut": "Ctrl+F4", + "tooltip": "Extract Urls From Text (Ctrl+F4)", + "function": "self.extract_url_from_raw_text", + "setCheckable": false, + "menu": "menu_tools", + "seperator": false, + "toolbar": "" + }, + { + "icon": "/icons/delete-outline.svg", + "name": "action_utilities_clean_output_folder", + "visible": true, + "text": "Clean &Output Directory", + "shortcut": "Ctrl+F5", + "tooltip": "Clean output directory (Ctrl+F5)", + "function": "self.clean_output_directory", + "setCheckable": false, + "menu": "menu_tools", + "seperator": false, + "toolbar": "" + }, + { + "icon": "/icons/help-box-outline.svg", + "name": "action_help_help", + "visible": true, + "text": "&Help", + "shortcut": "F1", + "tooltip": "Help", + "function": "self.help", + "setCheckable": false, + "menu": "menu_help", + "seperator": false, + "toolbar": "" + }, + { + "icon": "/icons/information-variant.svg", + "name": "action_help_about", + "visible": true, + "text": "&About", + "shortcut": "Ctrl+F1", + "tooltip": "About", + "function": "self.about", + "setCheckable": false, + "menu": "menu_help", + "seperator": true, + "toolbar": "" + } + ] +} \ No newline at end of file diff --git a/src/staticwordpress/share/icons/additionals.svg b/src/staticwordpress/share/icons/additionals.svg new file mode 100644 index 0000000..9d15628 --- /dev/null +++ b/src/staticwordpress/share/icons/additionals.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/bug-outline.svg b/src/staticwordpress/share/icons/bug-outline.svg new file mode 100644 index 0000000..98fe0b7 --- /dev/null +++ b/src/staticwordpress/share/icons/bug-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/check_project.svg b/src/staticwordpress/share/icons/check_project.svg new file mode 100644 index 0000000..7c74ab9 --- /dev/null +++ b/src/staticwordpress/share/icons/check_project.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/close-box-outline.svg b/src/staticwordpress/share/icons/close-box-outline.svg new file mode 100644 index 0000000..44dc673 --- /dev/null +++ b/src/staticwordpress/share/icons/close-box-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/cloud-upload-outline.svg b/src/staticwordpress/share/icons/cloud-upload-outline.svg new file mode 100644 index 0000000..431a5e8 --- /dev/null +++ b/src/staticwordpress/share/icons/cloud-upload-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/configs.svg b/src/staticwordpress/share/icons/configs.svg new file mode 100644 index 0000000..3195c64 --- /dev/null +++ b/src/staticwordpress/share/icons/configs.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/content-save-outline.svg b/src/staticwordpress/share/icons/content-save-outline.svg new file mode 100644 index 0000000..05e1e2e --- /dev/null +++ b/src/staticwordpress/share/icons/content-save-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/crawl_website.svg b/src/staticwordpress/share/icons/crawl_website.svg new file mode 100644 index 0000000..47dedcd --- /dev/null +++ b/src/staticwordpress/share/icons/crawl_website.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/delete-outline.svg b/src/staticwordpress/share/icons/delete-outline.svg new file mode 100644 index 0000000..372dd95 --- /dev/null +++ b/src/staticwordpress/share/icons/delete-outline.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/delete-repo.svg b/src/staticwordpress/share/icons/delete-repo.svg new file mode 100644 index 0000000..3391359 --- /dev/null +++ b/src/staticwordpress/share/icons/delete-repo.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/error.svg b/src/staticwordpress/share/icons/error.svg new file mode 100644 index 0000000..4154b45 --- /dev/null +++ b/src/staticwordpress/share/icons/error.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/exit-to-app.svg b/src/staticwordpress/share/icons/exit-to-app.svg new file mode 100644 index 0000000..46cfd07 --- /dev/null +++ b/src/staticwordpress/share/icons/exit-to-app.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/file-document-remove-outline.svg b/src/staticwordpress/share/icons/file-document-remove-outline.svg new file mode 100644 index 0000000..8004379 --- /dev/null +++ b/src/staticwordpress/share/icons/file-document-remove-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/file-outline.svg b/src/staticwordpress/share/icons/file-outline.svg new file mode 100644 index 0000000..e044f48 --- /dev/null +++ b/src/staticwordpress/share/icons/file-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/folder-git.svg b/src/staticwordpress/share/icons/folder-git.svg new file mode 100644 index 0000000..1e5fbe0 --- /dev/null +++ b/src/staticwordpress/share/icons/folder-git.svg @@ -0,0 +1,59 @@ + + + + + + + + + + + diff --git a/src/staticwordpress/share/icons/folder-off.svg b/src/staticwordpress/share/icons/folder-off.svg new file mode 100644 index 0000000..41993d2 --- /dev/null +++ b/src/staticwordpress/share/icons/folder-off.svg @@ -0,0 +1,39 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/folder-outline.svg b/src/staticwordpress/share/icons/folder-outline.svg new file mode 100644 index 0000000..a475181 --- /dev/null +++ b/src/staticwordpress/share/icons/folder-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/github.svg b/src/staticwordpress/share/icons/github.svg new file mode 100644 index 0000000..6d0f1bc --- /dev/null +++ b/src/staticwordpress/share/icons/github.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/help-box-outline.svg b/src/staticwordpress/share/icons/help-box-outline.svg new file mode 100644 index 0000000..e21a1a4 --- /dev/null +++ b/src/staticwordpress/share/icons/help-box-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/information-variant.svg b/src/staticwordpress/share/icons/information-variant.svg new file mode 100644 index 0000000..2109e64 --- /dev/null +++ b/src/staticwordpress/share/icons/information-variant.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/pencil-outline.svg b/src/staticwordpress/share/icons/pencil-outline.svg new file mode 100644 index 0000000..2e1d1ef --- /dev/null +++ b/src/staticwordpress/share/icons/pencil-outline.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/person.svg b/src/staticwordpress/share/icons/person.svg new file mode 100644 index 0000000..33468cf --- /dev/null +++ b/src/staticwordpress/share/icons/person.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/play.svg b/src/staticwordpress/share/icons/play.svg new file mode 100644 index 0000000..1977908 --- /dev/null +++ b/src/staticwordpress/share/icons/play.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/playlist-plus.svg b/src/staticwordpress/share/icons/playlist-plus.svg new file mode 100644 index 0000000..6a9f9a2 --- /dev/null +++ b/src/staticwordpress/share/icons/playlist-plus.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/redirects.svg b/src/staticwordpress/share/icons/redirects.svg new file mode 100644 index 0000000..7f778e0 --- /dev/null +++ b/src/staticwordpress/share/icons/redirects.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/robots_txt.svg b/src/staticwordpress/share/icons/robots_txt.svg new file mode 100644 index 0000000..ebbba1e --- /dev/null +++ b/src/staticwordpress/share/icons/robots_txt.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/search.svg b/src/staticwordpress/share/icons/search.svg new file mode 100644 index 0000000..762debc --- /dev/null +++ b/src/staticwordpress/share/icons/search.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/semantic-web.svg b/src/staticwordpress/share/icons/semantic-web.svg new file mode 100644 index 0000000..d554a19 --- /dev/null +++ b/src/staticwordpress/share/icons/semantic-web.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/staticwordpress/share/icons/static-wordpress.svg b/src/staticwordpress/share/icons/static-wordpress.svg new file mode 100644 index 0000000..c608a91 --- /dev/null +++ b/src/staticwordpress/share/icons/static-wordpress.svg @@ -0,0 +1,48 @@ + + + + diff --git a/src/staticwordpress/share/icons/stop.svg b/src/staticwordpress/share/icons/stop.svg new file mode 100644 index 0000000..1437318 --- /dev/null +++ b/src/staticwordpress/share/icons/stop.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/text-recognition.svg b/src/staticwordpress/share/icons/text-recognition.svg new file mode 100644 index 0000000..e8dd641 --- /dev/null +++ b/src/staticwordpress/share/icons/text-recognition.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/three-dots.svg b/src/staticwordpress/share/icons/three-dots.svg new file mode 100644 index 0000000..a8de325 --- /dev/null +++ b/src/staticwordpress/share/icons/three-dots.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/web-remove.svg b/src/staticwordpress/share/icons/web-remove.svg new file mode 100644 index 0000000..478d551 --- /dev/null +++ b/src/staticwordpress/share/icons/web-remove.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/src/staticwordpress/share/icons/wordpress.svg b/src/staticwordpress/share/icons/wordpress.svg new file mode 100644 index 0000000..d4b2e09 --- /dev/null +++ b/src/staticwordpress/share/icons/wordpress.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/staticwordpress/share/robots.txt b/src/staticwordpress/share/robots.txt new file mode 100644 index 0000000..10bb2b5 --- /dev/null +++ b/src/staticwordpress/share/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php \ No newline at end of file diff --git a/src/staticwordpress/share/search.js b/src/staticwordpress/share/search.js new file mode 100644 index 0000000..487b8d7 --- /dev/null +++ b/src/staticwordpress/share/search.js @@ -0,0 +1,307 @@ + +/* +The original source code of this file is avialble at +https://github.com/a-luna/aaronluna.dev/blob/master/static/js/search.js +There is no lincense string attached with original file/repository. + +static-wordpress [serpwings] also do not claim any license of this file. +Always check original source for updated Licensing information. +TODO: If you can help with creating better Search Representations, then +please open an issue to discuss it/implement it. +*/ + +let pagesIndex, searchIndex; +const MAX_SUMMARY_LENGTH = 200; +const SENTENCE_BOUNDARY_REGEX = /\b\.\s/gm; +const WORD_REGEX = /\b(\w*)[\W|\s|\b]?/gm; + +async function initSearchIndex() { + try { + const response = await fetch("./lunr.json"); + pagesIndex = await response.json(); + searchIndex = lunr(function () { + this.field("title"); + this.field("content"); + this.ref("href"); + pagesIndex.forEach((page) => this.add(page)); + }); + } catch (e) { + console.log(e); + } +} + +function searchBoxFocused() { + document.querySelector(".search-container").classList.add("focused"); + document + .getElementById("search") + .addEventListener("focusout", () => searchBoxFocusOut()); +} + +function searchBoxFocusOut() { + document.querySelector(".search-container").classList.remove("focused"); +} + +function handleSearchQuery(event) { + event.preventDefault(); + const query = document.getElementById("search").value.trim().toLowerCase(); + if (!query) { + displayErrorMessage("Please enter a search term"); + return; + } + const results = searchSite(query); + if (!results.length) { + displayErrorMessage("Your search returned no results"); + return; + } + renderSearchResults(query, results); +} + +function displayErrorMessage(message) { + document.querySelector(".search-error-message").innerHTML = message; + document.querySelector(".search-container").classList.remove("focused"); + document.querySelector(".search-error").classList.remove("hide-element"); + document.querySelector(".search-error").classList.add("fade"); +} + +function removeAnimation() { + this.classList.remove("fade"); + this.classList.add("hide-element"); + document.querySelector(".search-container").classList.add("focused"); +} + +function searchSite(query) { + const originalQuery = query; + query = getLunrSearchQuery(query); + let results = getSearchResults(query); + return results.length + ? results + : query !== originalQuery + ? getSearchResults(originalQuery) + : []; +} + +function getLunrSearchQuery(query) { + const searchTerms = query.split(" "); + if (searchTerms.length === 1) { + return query; + } + query = ""; + for (const term of searchTerms) { + query += `+${term} `; + } + return query.trim(); +} + +function getSearchResults(query) { + return searchIndex.search(query).flatMap((hit) => { + if (hit.ref == "undefined") return []; + let pageMatch = pagesIndex.filter((page) => page.href === hit.ref)[0]; + pageMatch.score = hit.score; + return [pageMatch]; + }); +} + +function renderSearchResults(query, results) { + clearSearchResults(); + updateSearchResults(query, results); + showSearchResults(); + scrollToTop(); +} + +function clearSearchResults() { + const results = document.querySelector(".search-results ul"); + while (results.firstChild) results.removeChild(results.firstChild); + + document.getElementById("query").innerHTML = ""; + + if (!results.length) { + displayErrorMessage(""); + return; + } +} + +function updateSearchResults(query, results) { + document.querySelector(".search-results ul").innerHTML = results + .map( + (hit) => ` +
  • + ${hit.title} +

    ${createSearchResultBlurb(query, hit.content)}

    +
    • + ` + ) + .join(""); + const searchResultListItems = document.querySelectorAll(".search-results ul li"); + document.getElementById("query").innerHTML = "Search Query: " + query + " (" + searchResultListItems.length + ")"; + searchResultListItems.forEach( + (li) => (li.firstElementChild.style.color = getColorForSearchResult(li.dataset.score)) + ); +} + +function createSearchResultBlurb(query, pageContent) { + const searchQueryRegex = new RegExp(createQueryStringRegex(query), "gmi"); + const searchQueryHits = Array.from( + pageContent.matchAll(searchQueryRegex), + (m) => m.index + ); + const sentenceBoundaries = Array.from( + pageContent.matchAll(SENTENCE_BOUNDARY_REGEX), + (m) => m.index + ); + let searchResultText = ""; + let lastEndOfSentence = 0; + for (const hitLocation of searchQueryHits) { + if (hitLocation > lastEndOfSentence) { + for (let i = 0; i < sentenceBoundaries.length; i++) { + if (sentenceBoundaries[i] > hitLocation) { + const startOfSentence = i > 0 ? sentenceBoundaries[i - 1] + 1 : 0; + const endOfSentence = sentenceBoundaries[i]; + lastEndOfSentence = endOfSentence; + parsedSentence = pageContent.slice(startOfSentence, endOfSentence).trim(); + searchResultText += `${parsedSentence} ... `; + break; + } + } + } + const searchResultWords = tokenize(searchResultText); + const pageBreakers = searchResultWords.filter((word) => word.length > 50); + if (pageBreakers.length > 0) { + searchResultText = fixPageBreakers(searchResultText, pageBreakers); + } + if (searchResultWords.length >= MAX_SUMMARY_LENGTH) break; + } + return ellipsize(searchResultText, MAX_SUMMARY_LENGTH).replace( + searchQueryRegex, + "$&" + ); +} + +function createQueryStringRegex(query) { + return query.split(" ").length == 1 ? `(${query})` : `(${query.split(" ").join("|")})`; +} + +function tokenize(input) { + const wordMatches = Array.from(input.matchAll(WORD_REGEX), (m) => m); + return wordMatches.map((m) => ({ + word: m[0], + start: m.index, + end: m.index + m[0].length, + length: m[0].length, + })); +} + +function fixPageBreakers(input, largeWords) { + largeWords.forEach((word) => { + const chunked = chunkify(word.word, 20); + input = input.replace(word.word, chunked); + }); + return input; +} + +function chunkify(input, chunkSize) { + let output = ""; + let totalChunks = (input.length / chunkSize) | 0; + let lastChunkIsUneven = input.length % chunkSize > 0; + if (lastChunkIsUneven) { + totalChunks += 1; + } + for (let i = 0; i < totalChunks; i++) { + let start = i * chunkSize; + let end = start + chunkSize; + if (lastChunkIsUneven && i === totalChunks - 1) { + end = input.length; + } + output += input.slice(start, end) + " "; + } + return output; +} + +function ellipsize(input, maxLength) { + const words = tokenize(input); + if (words.length <= maxLength) { + return input; + } + return input.slice(0, words[maxLength].end) + "..."; +} + +function showSearchResults() { + document.querySelector(".search-results").classList.add("hide-element"); +} + +function scrollToTop() { + const toTopInterval = setInterval(function () { + const supportedScrollTop = + document.body.scrollTop > 0 ? document.body : document.documentElement; + if (supportedScrollTop.scrollTop > 0) { + supportedScrollTop.scrollTop = supportedScrollTop.scrollTop - 50; + } + if (supportedScrollTop.scrollTop < 1) { + clearInterval(toTopInterval); + } + }, 10); +} + +function getColorForSearchResult(score) { + const warmColorHue = 171; + const coolColorHue = 212; + return adjustHue(warmColorHue, coolColorHue, score); +} + +function adjustHue(hue1, hue2, score) { + if (score > 3) return `hsl(${hue1}, 100%, 50%)`; + const hueAdjust = (parseFloat(score) / 3) * (hue1 - hue2); + const newHue = hue2 + Math.floor(hueAdjust); + return `hsl(${newHue}, 100%, 50%)`; +} + +function handleClearSearchButtonClicked() { + hideSearchResults(); + clearSearchResults(); + document.getElementById("search").value = ""; +} + +function hideSearchResults() { + document.querySelector(".search-results").classList.add("hide-element"); +} + +initSearchIndex(); +document.addEventListener("DOMContentLoaded", function () { + if (document.getElementById("search-form") != null) { + const searchInput = document.getElementById("search"); + searchInput.addEventListener("focus", () => searchBoxFocused()); + searchInput.addEventListener("keydown", (event) => { + if (event.keyCode == 13) handleSearchQuery(event); + }); + document + .querySelector(".search-error") + .addEventListener("animationend", removeAnimation); + } + document + .querySelectorAll(".fa-search") + .forEach((button) => + button.addEventListener("click", (event) => handleSearchQuery(event)) + ); + document + .querySelectorAll(".clear-search-results") + .forEach((button) => + button.addEventListener("click", () => handleClearSearchButtonClicked()) + ); +}); + +if (!String.prototype.matchAll) { + String.prototype.matchAll = function (regex) { + "use strict"; + function ensureFlag(flags, flag) { + return flags.includes(flag) ? flags : flags + flag; + } + function* matchAll(str, regex) { + const localCopy = new RegExp(regex, ensureFlag(regex.flags, "g")); + let match; + while ((match = localCopy.exec(str))) { + match.index = localCopy.lastIndex - match[0].length; + yield match; + } + } + return matchAll(this, regex); + }; +} \ No newline at end of file diff --git a/src/staticwordpress/share/translations.yaml b/src/staticwordpress/share/translations.yaml new file mode 100644 index 0000000..a0b04d6 --- /dev/null +++ b/src/staticwordpress/share/translations.yaml @@ -0,0 +1,6 @@ +Aborted!: + de_DE: Abgebrochen! +Cancel: + de_DE: Abbrechen +Close: + de_DE: Schließen \ No newline at end of file diff --git a/ss_script.py b/ss_script.py new file mode 100644 index 0000000..d44aa3a --- /dev/null +++ b/ss_script.py @@ -0,0 +1,76 @@ +#!/usr/bin/python + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + ss_script.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# STANDARD LIBARY IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +import os + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from src.staticwordpress.core.workflow import Workflow +from src.staticwordpress.core.constants import SOURCE, HOST + +if __name__ == "__main__": + env_wp_user = os.environ.get("user") + env_wp_api_token = os.environ.get("token") + env_src_url = os.environ.get("src") + env_dst_url = os.environ.get("dst") + + assert env_wp_user != "" + assert env_wp_api_token != "" + assert env_src_url != "" + assert env_dst_url != "" + + env_404 = os.environ.get("404") if os.environ.get("404") else "404-error" + env_search = os.environ.get("search") if os.environ.get("search") else "search" + env_output = os.environ.get("output") if os.environ.get("output") else "output" + + if not os.path.exists(env_output): + os.mkdir(env_output) + + ss_zip_obj = Workflow() + ss_zip_obj.create_project( + project_name_="simply-static-zip-deploy", + wp_user_=env_wp_user, + wp_api_token_=env_wp_api_token, + src_url_=env_src_url, + dst_url_=env_dst_url, + output_folder_=env_output, + custom_404_=env_404, + custom_search_=env_search, + src_type_=SOURCE.ZIP, + host_type_=HOST.NETLIFY, + ) + + ss_zip_obj.download_zip_file() + ss_zip_obj.setup_zip_folders() + ss_zip_obj.add_404_page() + ss_zip_obj.add_robots_txt() + ss_zip_obj.add_redirects() + ss_zip_obj.add_search() diff --git a/tests/test_project.py b/tests/test_project.py new file mode 100644 index 0000000..4c372f7 --- /dev/null +++ b/tests/test_project.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + tests\test_project.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from staticwordpress.core.project import Project +from staticwordpress.core.constants import PROJECT, REDIRECTS, HOST + + +def test_project_create(): + p = Project() + assert p.status == PROJECT.NOT_FOUND + assert p.redirects == REDIRECTS.REDIRECTION + assert p.host == HOST.NETLIFY diff --git a/tests/test_redirects.py b/tests/test_redirects.py new file mode 100644 index 0000000..c9b3ce5 --- /dev/null +++ b/tests/test_redirects.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + tests\test_redirects.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from staticwordpress.core.redirects import Redirects, Redirect +from staticwordpress.core.constants import REDIRECTS + + +def test_redirect(): + red = Redirect( + from_="/", + to_="https://seowings.org", + status=200, + query_=None, + force_=True, + source_=REDIRECTS.REDIRECTION, + ) + + assert red.as_line() == "/\thttps://seowings.org\t200" + assert red.as_json() == {"from": "/", "status": 200, "to": "https://seowings.org"} diff --git a/tests/test_translations.py b/tests/test_translations.py new file mode 100644 index 0000000..2ef88aa --- /dev/null +++ b/tests/test_translations.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + tests\test_translations.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from staticwordpress.core.i18n import tr +from staticwordpress.core.constants import LANGUAGES + + +def test_tranlsation_default_language(): + tr.language = LANGUAGES.en_US + assert tr.language == LANGUAGES.en_US + assert tr("Close") == "Close" + + +def test_tranlsation_german(): + tr.language = LANGUAGES.de_DE + assert tr.language == LANGUAGES.de_DE + assert tr("Close") == "Schließen" diff --git a/tests/test_url.py b/tests/test_url.py new file mode 100644 index 0000000..3cf7735 --- /dev/null +++ b/tests/test_url.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +""" +STATIC WORDPRESS: WordPress as Static Site Generator +A Python Package for Converting WordPress Installation to a Static Website +https://github.com/serpwings/staticwordpress + + tests\test_url.py + + Copyright (C) 2020-2023 Faisal Shahzad + + +The contents of this file are subject to version 3 of the +GNU General Public License (GPL-3.0). You may not use this file except in +compliance with the License. You may obtain a copy of the License at +https://www.gnu.org/licenses/gpl-3.0.txt +https://github.com/serpwings/staticwordpress/blob/master/LICENSE + + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +""" + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ +# INTERNAL IMPORTS +# +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +from staticwordpress.core.crawler import Crawler +from staticwordpress.core.constants import URL + + +def test_url_valid(): + my_url = Crawler(loc_="http://staticwp.local", type_=URL.HOME) + assert my_url.is_valid == True + + +def test_url_valid_2(): + my_url = Crawler(loc_="staticwp.local", type_=URL.HOME) + assert my_url.is_valid == True + + +def test_url_valid_3(): + my_url = Crawler(loc_="staticwp", type_=URL.HOME) + assert my_url.is_valid == False + + +def test_url_valid_4(): + my_url = Crawler(loc_="http://staticwp.local/test", type_=URL.FOLDER) + assert my_url.is_valid == True