commit 5e2a338031f319bd62af0230bb004dc212d58819 Author: tischrei Date: Wed Mar 5 11:05:51 2025 +0000 init diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f69ff5d --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True +source = otc-metadata + +[report] +ignore_errors = True diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9bff3c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,64 @@ +# Add patterns in here to exclude files created by tools integrated with this +# repository, such as test frameworks from the project's recommended workflow, +# rendered documentation and package builds. +# +# Don't add patterns to exclude files created by preferred personal tools +# (editors, IDEs, your operating system itself even). These should instead be +# maintained outside the repository, for example in a ~/.gitignore file added +# with: +# +# git config --global core.excludesfile '~/.gitignore' + +# Bytecompiled Python +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg* +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +cover/ +.coverage* +!.coveragerc +.tox +nosetests.xml +.testrepository +.stestr +.venv + +# Translations +*.mo + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Files created by releasenotes build +releasenotes/build + +bindep.txt +packages.txt \ No newline at end of file diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..f94068c --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.opendev.org +port=29418 +project=infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git.git diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..6b9459b --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./otc_metadata/tests/ +top_dir=./ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..e8e77d5 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,3 @@ +The source repository for this project can be found at: + + https://github.com/infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-rework.git diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..68c771a --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..bfb44b9 --- /dev/null +++ b/README.rst @@ -0,0 +1,54 @@ +============ +otc-metadata +============ + +Link: ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git + + +Metadata about OTC for Ecosystem + +Please fill here a long description which must be at least 3 lines wrapped on +80 cols, so that distribution package maintainers can use it in their packages. +Note that this is a hard requirement. + +* Free software: Apache license +* Documentation: https://docs.otc.t-systems.com/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git +* Source: https://github.com/infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git + +Features +======== + +* TODO + +Overview: service.yaml +====================== + +The :code:`service.yaml` file contains all data about services, service +categories and the related documents of each service. The file is +used as a base for several internal and external applications or +websites like the Helpcenter 3.0 where the information about the document +repositories and its properties are stored. + +File structure +-------------- + +The file is based on the yaml-file format and has three main sections +which can be compared with database tables in a relational database. + +* documents: contains the information about every single document and its type + like umn, api-ref etc. + +* service category: contains the keyword and title of the service category + +* services: contains the repository information about the internal (Gitea) and + external location (GitHub) and all the necessary parameters of the service itself + +These sections, or better "tables" have +their own keys and foreign keys so that the tables are linked together and +the related information can be fetched. +For the :code:`services` table +the key is :code:`service_type` which has the foreign key in the +:code:`documents` table. So a service can have multiple documents and each +document can only be linked to one service. +The key :code:`service_category` table is :code:`name` of the service category +which is then used in the :code:`services` table as foreign key. diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..23b871f --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,4 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +otcdocstheme>=1.0.0 # Apache-2.0 +# releasenotes +reno>=3.1.0 # Apache-2.0 diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst new file mode 100644 index 0000000..baa1cdb --- /dev/null +++ b/doc/source/admin/index.rst @@ -0,0 +1,5 @@ +==================== +Administrators guide +==================== + +Administrators guide of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git. diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst new file mode 100644 index 0000000..fc0360f --- /dev/null +++ b/doc/source/cli/index.rst @@ -0,0 +1,5 @@ +================================ +Command line interface reference +================================ + +CLI reference of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git. diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100755 index 0000000..3c78326 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'otcdocstheme', + #'sphinx.ext.intersphinx', +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git' +copyright = '2022, Open Telekom Cloud Developers' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] +html_theme = 'otcdocs' + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + '%s Documentation' % project, + 'Open Telekom Cloud Developers', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst new file mode 100644 index 0000000..230a71c --- /dev/null +++ b/doc/source/configuration/index.rst @@ -0,0 +1,5 @@ +============= +Configuration +============= + +Configuration of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git. diff --git a/doc/source/contributor/contributing.rst b/doc/source/contributor/contributing.rst new file mode 100644 index 0000000..2e91325 --- /dev/null +++ b/doc/source/contributor/contributing.rst @@ -0,0 +1,5 @@ +============================ +So You Want to Contribute... +============================ + +TODO diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 0000000..036e449 --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1,9 @@ +=========================== + Contributor Documentation +=========================== + +.. toctree:: + :maxdepth: 2 + + contributing + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..deb017d --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,30 @@ +.. ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +============================================ +Welcome to the documentation of otc-metadata +============================================ + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + install/index + library/index + contributor/index + configuration/index + cli/index + user/index + admin/index + reference/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst new file mode 100644 index 0000000..6d7dc44 --- /dev/null +++ b/doc/source/install/index.rst @@ -0,0 +1,14 @@ +================================================ + {{cookiecutter.module_name}} installation guide +================================================ + +.. toctree:: + :maxdepth: 2 + + install.rst + +The otc-metadata provides... + +This chapter assumes a working setup of OpenStack following the +`OpenStack Installation Tutorial +`_. diff --git a/doc/source/install/install.rst b/doc/source/install/install.rst new file mode 100644 index 0000000..3d5c5a7 --- /dev/null +++ b/doc/source/install/install.rst @@ -0,0 +1,13 @@ +.. _install: + +Install and configure +~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the +otc-metadata. + +This section assumes that you already have a working OpenStack +environment with at least the following components installed: +.. (add the appropriate services here and further notes) + +Note that installation and configuration vary by distribution. diff --git a/doc/source/library/index.rst b/doc/source/library/index.rst new file mode 100644 index 0000000..bd783d1 --- /dev/null +++ b/doc/source/library/index.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git in a project:: + + import otc-metadata diff --git a/doc/source/readme.rst b/doc/source/readme.rst new file mode 100644 index 0000000..a6210d3 --- /dev/null +++ b/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst new file mode 100644 index 0000000..84c158a --- /dev/null +++ b/doc/source/reference/index.rst @@ -0,0 +1,5 @@ +========== +References +========== + +References of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git. diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..410ec21 --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,5 @@ +=========== +Users guide +=========== + +Users guide of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata.git. diff --git a/otc_metadata/__init__.py b/otc_metadata/__init__.py new file mode 100644 index 0000000..3b458c0 --- /dev/null +++ b/otc_metadata/__init__.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +__all__ = ["__version__", "Docs"] + +import pbr.version + +from otc_metadata.services import Services # flake8: noqa + +__version__ = pbr.version.VersionInfo("otc-metadata").version_string() +_service_manager = None + + +def get_service_data(*args, **kwargs): + """Return singleton instance of the Services object. + Parameters are all passed through to the + :class:`~otc_metadata.services.Services` constructor. + .. note:: + Only one singleton is kept, so if instances with different parameter + values are desired, directly calling the constructor is necessary. + :returns: Singleton instance of + :class:`~otc_metadata.services.Services` + """ + global _service_manager + if not _service_manager: + _service_manager = Services(*args, **kwargs) + return _service_manager diff --git a/otc_metadata/data/__init__.py b/otc_metadata/data/__init__.py new file mode 100644 index 0000000..60b51a1 --- /dev/null +++ b/otc_metadata/data/__init__.py @@ -0,0 +1,56 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pathlib +import yaml + +__all__ = ["read_data"] + +DATA_DIR = os.path.dirname(__file__) + + +def read_data(filename): + """Return data that is shipped inside the Python package.""" + + filepath = os.path.join(DATA_DIR, filename) + with open(filepath, "r") as fd: + data = yaml.safe_load(fd) + # Merge data found in individual element files + data.setdefault("documents", list()) + data.setdefault("services", list()) + data.setdefault("service_categories", list()) + for item in pathlib.Path(DATA_DIR, "documents").glob("*.yaml"): + with open(item, "r") as fp: + data["documents"].append(yaml.safe_load(fp)) + for item in pathlib.Path(DATA_DIR, "services").glob("*.yaml"): + with open(item, "r") as fp: + data["services"].append(yaml.safe_load(fp)) + for item in pathlib.Path(DATA_DIR, "service_categories").glob( + "*.yaml" + ): + with open(item, "r") as fp: + data["service_categories"].append(yaml.safe_load(fp)) + + return data + + +def rewrite_data(filename, data): + """Rewrites data formatting it""" + from ruamel.yaml import YAML + + _yaml = YAML() + _yaml.indent(mapping=2, sequence=4, offset=2) + filepath = os.path.join(DATA_DIR, filename) + with open(filepath, "w") as fd: + _yaml.dump(data, fd) diff --git a/otc_metadata/data/documents/as-api-ref.yaml b/otc_metadata/data/documents/as-api-ref.yaml new file mode 100644 index 0000000..e92294a --- /dev/null +++ b/otc_metadata/data/documents/as-api-ref.yaml @@ -0,0 +1,10 @@ +--- +hc_location: api/as +html_location: docs/as/api-ref +link: /auto-scaling/api-ref/ +pdf_enabled: true +pdf_environment: internal +rst_location: api-ref/source +service_type: as +title: API Reference +type: api-ref diff --git a/otc_metadata/data/documents/as-dev.yaml b/otc_metadata/data/documents/as-dev.yaml new file mode 100644 index 0000000..48f29e6 --- /dev/null +++ b/otc_metadata/data/documents/as-dev.yaml @@ -0,0 +1,10 @@ +--- +hc_location: devg/as +html_location: docs/as/dev +link: /auto-scaling/dev-guide/ +pdf_enabled: true +pdf_environment: internal +rst_location: dev_guide/source +service_type: as +title: Developer Guide +type: dev diff --git a/otc_metadata/data/documents/as-umn.yaml b/otc_metadata/data/documents/as-umn.yaml new file mode 100644 index 0000000..b3fdec3 --- /dev/null +++ b/otc_metadata/data/documents/as-umn.yaml @@ -0,0 +1,10 @@ +--- +hc_location: usermanual/as +html_location: docs/as/umn +link: /auto-scaling/umn/ +pdf_enabled: true +pdf_environment: internal +rst_location: umn/source +service_type: as +title: User Guide +type: umn diff --git a/otc_metadata/data/repositories/as.yaml b/otc_metadata/data/repositories/as.yaml new file mode 100644 index 0000000..c71d42e --- /dev/null +++ b/otc_metadata/data/repositories/as.yaml @@ -0,0 +1,23 @@ +--- +service_type: as +- environment: public + repo: opentelekomcloud-docs/auto-scaling + type: github + cloud_environments: + - eu_de +- environment: internal + repo: docs/auto-scaling + type: gitea + - eu_de +- environment: public + repo: opentelekomcloud-docs-swiss/auto-scaling + type: github + cloud_environments: + - swiss +- environment: internal + repo: docs-swiss/auto-scaling + type: gitea + cloud_environments: + - swiss + + diff --git a/otc_metadata/data/service_categories/application.yaml b/otc_metadata/data/service_categories/application.yaml new file mode 100644 index 0000000..54dcbe2 --- /dev/null +++ b/otc_metadata/data/service_categories/application.yaml @@ -0,0 +1,3 @@ +--- +name: application +title: Application Services diff --git a/otc_metadata/data/service_categories/big_data.yaml b/otc_metadata/data/service_categories/big_data.yaml new file mode 100644 index 0000000..5951c5f --- /dev/null +++ b/otc_metadata/data/service_categories/big_data.yaml @@ -0,0 +1,3 @@ +--- +name: big_data +title: Big Data and Data Analysis diff --git a/otc_metadata/data/service_categories/compute.yaml b/otc_metadata/data/service_categories/compute.yaml new file mode 100644 index 0000000..c3dee8f --- /dev/null +++ b/otc_metadata/data/service_categories/compute.yaml @@ -0,0 +1,3 @@ +--- +name: compute +title: Computing diff --git a/otc_metadata/data/service_categories/container.yaml b/otc_metadata/data/service_categories/container.yaml new file mode 100644 index 0000000..90f6142 --- /dev/null +++ b/otc_metadata/data/service_categories/container.yaml @@ -0,0 +1,3 @@ +--- +name: container +title: Container diff --git a/otc_metadata/data/service_categories/database.yaml b/otc_metadata/data/service_categories/database.yaml new file mode 100644 index 0000000..fda95dc --- /dev/null +++ b/otc_metadata/data/service_categories/database.yaml @@ -0,0 +1,3 @@ +--- +name: database +title: Databases diff --git a/otc_metadata/data/service_categories/md.yaml b/otc_metadata/data/service_categories/md.yaml new file mode 100644 index 0000000..ad7ccc3 --- /dev/null +++ b/otc_metadata/data/service_categories/md.yaml @@ -0,0 +1,3 @@ +--- +name: md +title: Management & Deployment diff --git a/otc_metadata/data/service_categories/network.yaml b/otc_metadata/data/service_categories/network.yaml new file mode 100644 index 0000000..bd0a98b --- /dev/null +++ b/otc_metadata/data/service_categories/network.yaml @@ -0,0 +1,3 @@ +--- +name: network +title: Networking diff --git a/otc_metadata/data/service_categories/other.yaml b/otc_metadata/data/service_categories/other.yaml new file mode 100644 index 0000000..d5702a9 --- /dev/null +++ b/otc_metadata/data/service_categories/other.yaml @@ -0,0 +1,3 @@ +--- +name: other +title: Other diff --git a/otc_metadata/data/service_categories/security-services.yaml b/otc_metadata/data/service_categories/security-services.yaml new file mode 100644 index 0000000..dd0eae6 --- /dev/null +++ b/otc_metadata/data/service_categories/security-services.yaml @@ -0,0 +1,3 @@ +--- +name: security-services +title: Security Services diff --git a/otc_metadata/data/service_categories/storage.yaml b/otc_metadata/data/service_categories/storage.yaml new file mode 100644 index 0000000..113a4f0 --- /dev/null +++ b/otc_metadata/data/service_categories/storage.yaml @@ -0,0 +1,3 @@ +--- +name: storage +title: Storage diff --git a/otc_metadata/data/services.yaml b/otc_metadata/data/services.yaml new file mode 100644 index 0000000..ba5e83a --- /dev/null +++ b/otc_metadata/data/services.yaml @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please consult with otc_metadata.services.Services:_sort_data for expected +# sort order. When unsure perform tools/sort_metadata.py for getting it sorted. +# It is also supported, that content of files under +# documents/services/service_categories is being merged with +# the content here. +--- +documents: [] +service_categories: [] +services: [] diff --git a/otc_metadata/data/services/as.yaml b/otc_metadata/data/services/as.yaml new file mode 100644 index 0000000..19b10c9 --- /dev/null +++ b/otc_metadata/data/services/as.yaml @@ -0,0 +1,13 @@ +--- +environment: public +repositories: +service_category: compute +service_title: Auto Scaling +service_type: as +service_uri: auto-scaling +cloud_environments: +- name: eu_de +- name: swiss +teams: +- name: docs-compute-rw + permission: write diff --git a/otc_metadata/docs.py b/otc_metadata/docs.py new file mode 100644 index 0000000..d1d7c63 --- /dev/null +++ b/otc_metadata/docs.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +import otc_metadata.data + +__all__ = ["Service"] + +BUILTIN_DATA = otc_metadata.data.read_data("docs.yaml") + + +def _normalize_type(service_type): + if service_type: + return service_type.replace("_", "-") + + +class Service(object): + """Encapsulation of the OTC Docs data""" + + def __init__(self): + self._service_data = BUILTIN_DATA + + @property + def all_services(self): + "Service Categories data listing." + return copy.deepcopy(self._service_data["services"]) diff --git a/otc_metadata/services.py b/otc_metadata/services.py new file mode 100644 index 0000000..ced16fd --- /dev/null +++ b/otc_metadata/services.py @@ -0,0 +1,308 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import warnings + +import otc_metadata.data + +__all__ = ["Services"] + +BUILTIN_DATA = otc_metadata.data.read_data("services.yaml") + + +def _normalize_type(service_type): + if service_type: + return service_type.replace("_", "-") + + +class Services(object): + """Encapsulation of the OTC Services data""" + + def __init__(self): + self._service_data = BUILTIN_DATA + + def _sort_data(self): + # Sort every doc item by key + sorted_docs = [] + for doc in self._service_data["documents"]: + sorted_docs.append(dict(sorted(doc.items(), key=lambda kv: kv[0]))) + + # sort docs list by _ + self._service_data["documents"] = sorted( + sorted_docs, + key=lambda x: f"{x.get('service_type')}{x.get('title')}", + ) + # sort services by <service_type>_<service_title> + self._service_data["services"] = sorted( + self._service_data["services"], + key=lambda x: f"{x.get('service_type')}{x.get('service_title')}", + ) + # sort service categories by <name>_<title> + self._service_data["service_categories"] = sorted( + self._service_data["service_categories"], + key=lambda x: f"{x.get('name')}{x.get('title')}", + ) + other = {'name': 'other', 'title': 'Other'} + if other in self._service_data["service_categories"]: + self._service_data["service_categories"].remove(other) + self._service_data["service_categories"].append(other) + + def _rewrite_data(self): + otc_metadata.data.rewrite_data("services.yaml", self._service_data) + + @property + def all_services(self): + "Service Categories data listing." + return copy.deepcopy(self._service_data["services"]) + + @property + def all_docs(self): + "Service Docs data listing." + return copy.deepcopy(self._service_data["documents"]) + + @property + def service_dict(self): + "Service Docs data listing." + res = dict() + for srv in self.all_services: + res[srv["service_type"]] = copy.deepcopy(srv) + return res + + @property + def service_categories(self): + """List services categories""" + res = [] + for cat in self._service_data["service_categories"]: + res.append(copy.deepcopy(cat)) + return res + + def services_by_category(self, category, environment=None): + """List services matching category""" + res = [] + for srv in self.all_services: + if environment: + if "environment" in srv and srv["environment"] != environment: + continue + if srv["service_category"] == category: + res.append(copy.deepcopy(srv)) + return res + + def services_with_docs_by_category(self, category, environment=None): + """Retrieve service category docs data + + :param str category: Optional Category filter + :param str env: Optional service environment. Influeces "repository" + field + """ + res = dict() + services = self.service_dict + for doc in self.all_docs: + cat = doc["service_type"] + service = services.get(cat) + if not service: + warnings.warn("No Service defition of type %s" % (cat)) + continue + if category and service["service_category"] != category: + continue + res.setdefault(cat, service) + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + if "environment" in doc and doc["environment"] != environment: + continue + res[cat].setdefault("docs", []) + res[cat]["docs"].append(res_doc) + return res + + def service_types_with_doc_types(self, environment=None): + """Retrieve type and title from services and corresponding docs. + As well as a list of all available doc types with title. + + :param str environment: Optional service environment. + """ + service_list = [] + docs = [] + + for service in self.all_services: + if "environment" in service: + if service["environment"] != environment: + continue + if not service["service_title"]: + continue + if not service["service_type"]: + continue + + doc_list = [] + for doc in self.all_docs: + if "environment" in doc: + if doc["environment"] != environment: + continue + if doc["service_type"] == service["service_type"]: + doc_list.append({ + "title": doc["title"], + "type": doc["type"] + }) + + new_doc = { + "type": doc["type"], + "title": doc["title"] + } + type_exists = any( + doc_dict["type"] == new_doc["type"] for doc_dict in docs + ) + if not type_exists: + docs.append(new_doc) + + service_list.append({ + "service_title": service["service_title"], + "service_type": service["service_type"], + "docs": doc_list + }) + + res = { + "services": service_list, + "docs": docs + } + + return res + + def docs_by_service_category(self, category, environment=None): + """List services matching category + + :param str category: Category name + :param str env: Optional service environment. Influeces "repository" + field + """ + res = [] + services = self.service_dict + for doc in self.all_docs: + cat = doc["service_type"] + service = services.get(cat) + if not service: + warnings.warn("No Service defition of type %s" % (cat)) + continue + if service["service_category"] == category: + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + for srv_env in service["repositories"]: + if srv_env.get("environment") == environment: + res_doc["repository"] = srv_env["repo"] + res.append(res_doc) + return res + + def docs_by_service_type(self, service_type): + """List documents of the service + + :param str service_type: Service type + :returns: generator for documents + """ + for doc in self.all_docs: + if doc["service_type"] != service_type: + continue + yield copy.deepcopy(doc) + + def all_docs_full(self, environment): + """Return list or documents with full service data""" + services = self.service_dict + for doc in self.all_docs: + if not doc["service_type"] in services: + print(f"No service type {doc['service_type']}") + continue + service = services[doc["service_type"]] + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + for srv_env in service["repositories"]: + if srv_env.get("environment") == environment: + res_doc["repository"] = srv_env["repo"] + for srv_assignees in service.get("assignees", []): + if srv_assignees.get("environment") == environment: + res_doc["assignees"] = srv_assignees["names"] + yield res_doc + + def docs_html_by_category(self, environment): + """Generate structure for doc-exports repository""" + doc_struct = dict() + for srv in self.all_services: + doc_struct.setdefault(srv["service_category"], []) + srv_res = dict( + service_title=srv["service_title"], + service_type=srv["service_type"], + service_category=srv["service_category"], + service_environment=environment, + docs=[], + ) + if "teams" in srv: + srv_res["teams"] = [ + x for x in srv["teams"] if x["permission"] == "write" + ] + if "repositories" in srv and environment: + internal_exists = False + for repo in srv["repositories"]: + if ( + "environment" in repo + and repo["environment"] == environment + ): + srv_res["repository"] = repo["repo"] + if repo["environment"] == "internal": + internal_exists = True + # internal repo does not exist + # service will be left out from metadata.yaml + if not internal_exists: + continue + for doc in self.all_docs: + if ( + "html_location" in doc + and doc["service_type"] == srv_res["service_type"] + ): + doc_res = dict( + html_location=doc["html_location"], + rst_location=doc["rst_location"], + title=doc["title"], + type=doc.get("type", "dummy"), + link=doc["link"], + ) + if "pdf_name" in doc: + doc_res["pdf_name"] = doc["pdf_name"] + if "hc_location" in doc: + doc_res["hc_location"] = doc["hc_location"] + if "disable_import" in doc: + doc_res["disable_import"] = doc["disable_import"] + else: + doc_res["disable_import"] = False + srv_res["docs"].append(doc_res) + if len(srv_res["docs"]) > 0: + doc_struct[srv["service_category"]].append(srv_res) + + return dict(categories=doc_struct) + + def get_service_with_docs_by_service_type(self, service_type): + """Retrieve service and service docs by service_type + + :param str service_type: Filter by service_type + """ + res = dict() + res["service"] = {} + docs = [] + services = self._service_data + for doc in services["documents"]: + if doc["service_type"] == service_type: + docs.append(doc) + res["documents"] = docs + for service in services["services"]: + if service["service_type"] == service_type: + res["service"] = service + break + return res diff --git a/otc_metadata/templates/conf.py.j2 b/otc_metadata/templates/conf.py.j2 new file mode 100644 index 0000000..e98e7cd --- /dev/null +++ b/otc_metadata/templates/conf.py.j2 @@ -0,0 +1,158 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# !!! +# This file is generated out of template in doc-exports repository. +# Beware overwriting it locally. + +import os +import sys +from git import Repo +from datetime import datetime + +extensions = [ + 'otcdocstheme', +{%- if otc_sbv %} + 'otc_sphinx_directives' +{%- endif %} +] + +otcdocs_auto_name = False +otcdocs_auto_version = False + +project = '{{ project }}' +otcdocs_repo_name = '{{ repo_name }}' +# Those variables are required for edit/bug links +{%- if git_fqdn %} +otcdocs_git_fqdn = '{{ git_fqdn }}' +{%- endif %} +{%- if git_type %} +otcdocs_git_type = '{{ git_type }}' +{%- endif %} + +# Those variables are needed for indexing into OpenSearch +otcdocs_doc_environment = '{{ doc_environment }}' +otcdocs_doc_link = '{{ doc_link }}' +otcdocs_doc_title = '{{ doc_title }}' +otcdocs_doc_type = '{{ doc_type }}' +otcdocs_service_category = '{{ service_category }}' +otcdocs_service_title = '{{ service_title }}' +otcdocs_service_type = '{{ service_type }}' +otcdocs_search_environment = 'hc_de' +otcdocs_search_url = "https://opensearch.eco.tsi-dev.otc-service.com/" + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) + +# -- General configuration ---------------------------------------------------- +# https://docutils.sourceforge.io/docs/user/smartquotes.html - it does not +# what it is expected +smartquotes = False + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +copyright = u'2022-present, Open Telekom Cloud' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +language = 'en' + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { +{%- if html_options is defined -%} +{%- for (k, v) in html_options.items() %} +{%- if v is boolean %} + "{{ k }}": {{ v }}, +{%- else %} + "{{ k }}": "{{ v }}", +{%- endif %} +{%- endfor %} +{%- endif %} +} + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +{% if title is defined %} +html_title = "{{ title }}" +{% endif %} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +templates_path = ['_templates'] + +# Do not include sources into the rendered results +html_copy_source = False + +# -- Options for PDF output -------------------------------------------------- +latex_documents = [ +{%- if doc_type %} + ('index', + {%- if doc_type == 'dev' %} + '{{ service_type }}-dev-guide.tex', + {%- else %} + '{{ service_type }}-{{ doc_type }}.tex', + {% endif -%} + u'{{ title }}', + u'OpenTelekomCloud', 'manual'), +{% endif -%} +] + +# Get the Git commit values for last updated timestamp on each page +repo = Repo(search_parent_directories=True) +commit = repo.head.commit +current_commit_hash = commit.hexsha +current_commit_time = commit.committed_datetime.strftime('%Y-%m-%d %H:%M') + +latex_elements = { + 'papersize': 'a4paper', + 'pointsize': '12pt', + 'figure_align': 'H', + 'preamble': rf'''{% raw %} + \newcommand{{\githash}}{{{current_commit_hash}}} + \newcommand{{\gitcommittime}}{{{current_commit_time}}} + \newcommand{{\doctitle}}{{{otcdocs_doc_title}}} + \newcommand{{\servicetitle}}{{{otcdocs_service_title}}}{% endraw %} + ''', + 'sphinxsetup': 'hmargin={15mm,15mm}, vmargin={20mm,30mm}, marginpar=10mm' +} diff --git a/otc_metadata/templates/doc_requirements.txt.j2 b/otc_metadata/templates/doc_requirements.txt.j2 new file mode 100644 index 0000000..61cdd1b --- /dev/null +++ b/otc_metadata/templates/doc_requirements.txt.j2 @@ -0,0 +1,16 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +{% if target_environment == 'public' %} +otcdocstheme<2.0.0 # Apache-2.0 +{% elif target_environment == 'internal' %} +otcdocstheme # Apache-2.0 +{% else %} +otcdocstheme # Apache-2.0 +{% endif %} +# releasenotes +reno>=3.1.0 # Apache-2.0 + +otc-sphinx-directives>=0.1.0 +sphinx-minify>=0.0.1 # Apache-2.0 +git+https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata.git#egg=otc_metadata +setuptools +gitpython diff --git a/otc_metadata/templates/index_sbv.rst.j2 b/otc_metadata/templates/index_sbv.rst.j2 new file mode 100644 index 0000000..e30cc14 --- /dev/null +++ b/otc_metadata/templates/index_sbv.rst.j2 @@ -0,0 +1,8 @@ +{{ sbv_title }} + +.. directive_wrapper:: + :class: container-sbv + + .. service_card:: + :service_type: {{ service_type }} + :environment: {{ environment }} diff --git a/otc_metadata/templates/tox.ini.j2 b/otc_metadata/templates/tox.ini.j2 new file mode 100644 index 0000000..edfd5f1 --- /dev/null +++ b/otc_metadata/templates/tox.ini.j2 @@ -0,0 +1,151 @@ +[tox] +minversion = 3.1 +envlist = py39,pep8 +skipsdist = True +ignore_basepython_conflict = True + +[testenv] +usedevelop = True +install_command = pip install {opts} {packages} +deps = + -r{toxinidir}/requirements.txt +commands = stestr run {posargs} + stestr slowest + +[testenv:pep8] +allowlist_externals = + doc8 +commands = + doc8 doc/source README.rst + +[testenv:venv] +deps = + -r{toxinidir}/requirements.txt +commands = {posargs} + +# This env is invoked in the periodic pipeline and is therefore responsible to +# build all relevant docs at once. +[testenv:docs] +deps = + -r{toxinidir}/doc/requirements.txt + -c https://raw.githubusercontent.com/opentelekomcloud-docs/docs-constraints/main/constraints.txt +allowlist_externals = + mkdir + cp + sh + rm + sphinx-build +commands = + rm -rf doc/build/html doc/build/html_temp doc/build/doctrees + sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html_temp + sphinx-minify --input-directory doc/build/html_temp/ --output-directory doc/build/html +{%- for doc in docs %} + {[testenv:{{ doc.type }}]commands} + {[testenv:json-{{ doc.type }}]commands} +{%- endfor %} + +{% if docs|length > 0 %} +[testenv:pdf-docs] +deps = + {[testenv:docs]deps} + {[testenv:bindeps]deps} +allowlist_externals = + rm + mkdir + wget + make + bash + cp +commands = + mkdir -p doc/build/pdf + mkdir -p doc/build/html + {[testenv:bindeps]commands} + mkdir -p {toxinidir}/_templates + wget -O {toxinidir}/_templates/longtable.tex.jinja https://gitea.eco.tsi-dev.otc-service.com/infra/docs-templates/raw/branch/main/templates/longtable.tex.jinja + wget -O {toxinidir}/_templates/tabular.tex.jinja https://gitea.eco.tsi-dev.otc-service.com/infra/docs-templates/raw/branch/main/templates/tabular.tex.jinja + wget -O {toxinidir}/_templates/tabulary.tex.jinja https://gitea.eco.tsi-dev.otc-service.com/infra/docs-templates/raw/branch/main/templates/tabulary.tex.jinja +{%- for doc in docs %} +{%- if doc.pdf_enabled %} + {[testenv:{{ doc.type }}-pdf-docs]commands} +{%- endif %} +{%- endfor %} +{% endif %} + +{% for doc in docs -%} +{% set loc = doc.rst_location | replace('/source', '') %} +# HTML version +[testenv:{{ doc.type }}] +deps = {[testenv:docs]deps} +allowlist_externals = + cp + mkdir +commands = + sphinx-build -W --keep-going -b html {{ loc }}/source doc/build/html_temp/{{ doc.type }} + sphinx-minify --input-directory doc/build/html_temp/{{ doc.type }} --output-directory doc/build/html/{{ doc.type }} +{%- if doc.type == 'dev-guide' %} + mkdir -p dev_guide/build/html + cp -av doc/build/html/dev-guide dev_guide/build/html +{%- else %} + mkdir -p {{ doc.type }}/build/html + cp -av doc/build/html/{{ doc.type }} {{ doc.type }}/build/html +{%- endif %} + +# Json version (for search) +[testenv:json-{{ doc.type }}] +deps = {[testenv:docs]deps} +allowlist_externals = + cp + mkdir + sh +commands = + sphinx-build -W --keep-going -b json {{ loc }}/source doc/build/json/{{ doc.type }} + # Drop data useless for the search - wrap it also with sh/xargs due to bugs + # in tox + sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf" +{%- if doc.type == 'dev-guide' %} + mkdir -p dev_guide/build/json + cp -av doc/build/json/dev-guide dev_guide/build/json +{%- else %} + mkdir -p {{ doc.type }}/build/json + cp -av doc/build/json/{{ doc.type }} {{ doc.type }}/build/json +{%- endif %} + +# PDF version +[testenv:{{ doc.type }}-pdf-docs] +deps = {[testenv:docs]deps} +allowlist_externals = + rm + mkdir + make + bash + cp +commands = + rm -rf {{ loc }}/build/pdf + cp -r {toxinidir}/_templates {{ loc }}/source/_templates/ + sphinx-build -W --keep-going -b latex {{ loc }}/source {{ loc }}/build/pdf/ + bash -c "for f in {{ loc }}/build/pdf/*.gif; do convert $f[0] $\{f/%gif/png\}; done || true" + bash -c "for f in {{ loc }}/build/pdf/*.tex; do sed -iorig 's/\.gif//g' $f; done" + make -C {{ loc }}/build/pdf LATEXMKOPTS="-interaction=nonstopmode" + mkdir -p doc/build/pdf + cp {{ loc }}/build/pdf/{{ service_type }}-{{ doc.type }}.pdf doc/build/pdf/ + cp {{ loc }}/build/pdf/{{ service_type }}-{{ doc.type }}.pdf doc/build/html/ +{% endfor %} + +[testenv:bindeps] +deps = + bindep +allowlist_externals = + wget + rm + bash +commands = + rm -rf {toxinidir}/bindep.txt + rm -rf {toxinidir}/packages.txt + wget -O {toxinidir}/bindep.txt https://raw.githubusercontent.com/opentelekomcloud/otcdocstheme/main/bindep.txt + bash -c "bindep test -b -f {toxinidir}/bindep.txt > {toxinidir}/packages.txt || true" + bash -c 'if [ -s {toxinidir}/packages.txt ]; then if command -v apt &>/dev/null; then apt update && xargs apt install --no-install-recommends -y < {toxinidir}/packages.txt; fi; fi' + bash -c 'if [ -s {toxinidir}/packages.txt ]; then if command -v dnf &>/dev/null; then dnf install -y $(cat {toxinidir}/packages.txt); fi; fi' + +[doc8] +ignore = D001 +extensions = .rst, .yaml \ No newline at end of file diff --git a/otc_metadata/templates/zuul.yaml.j2 b/otc_metadata/templates/zuul.yaml.j2 new file mode 100644 index 0000000..f0f6112 --- /dev/null +++ b/otc_metadata/templates/zuul.yaml.j2 @@ -0,0 +1,19 @@ +--- +- project: + merge-mode: squash-merge + default-branch: main + templates: + - helpcenter-base-jobs + vars: + sphinx_pdf_files: + {%- for doc in docs %} + {%- if doc.pdf_enabled %} + - {{ service_type }}-{{ doc.type }}.pdf + {%- endif %} + {%- endfor %} + check: + jobs: + - noop + gate: + jobs: + - noop \ No newline at end of file diff --git a/otc_metadata/tests/__init__.py b/otc_metadata/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/otc_metadata/tests/test_otc_metadata.py b/otc_metadata/tests/test_otc_metadata.py new file mode 100644 index 0000000..6820c6d --- /dev/null +++ b/otc_metadata/tests/test_otc_metadata.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +test_otc-metadata +---------------------------------- + +Tests for `otc-metadata` module. +""" + +from unittest import TestCase + +from otc_metadata import services + + +class TestOtcMetadata(TestCase): + def setUp(self): + self.data = services.Services() + + def test_data_is_sorted(self): + curr = self.data + new = services.Services() + new._sort_data() + self.assertEqual( + curr._service_data, new._service_data, "Data is sorted properly" + ) + + def test_service_categories(self): + category = dict() + for cat in self.data._service_data["service_categories"]: + category[cat["name"]] = cat["title"] + for srv in self.data.all_services: + self.assertTrue( + srv["service_category"] in category, + f"Category {srv['service_category']} is present", + ) + self.assertGreater( + len(self.data._service_data["service_categories"]), + 1, + "More then 1 service category", + ) + + def test_doc_contains_required_data(self): + srv_types = dict() + for srv in self.data.all_services: + srv_types[srv["service_type"]] = srv + for doc in self.data.all_docs: + for attr in [ + "rst_location", + "service_type", + "title", + "type", + ]: + self.assertIn(attr, doc, f"Document {doc} contains {attr}") + self.assertIn( + doc["service_type"], + srv_types, + f"Document {doc} contains valid service_type", + ) diff --git a/releasenotes/notes/.placeholder b/releasenotes/notes/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/_static/.placeholder b/releasenotes/source/_static/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/_templates/.placeholder b/releasenotes/source/_templates/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 0000000..ecd0d77 --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'otcdocstheme', + 'reno.sphinxext', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'otc-metadata Release Notes' +copyright = '2022, Open Telekom Cloud Developers' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +# The full version, including alpha/beta/rc tags. +release = '' +# The short X.Y version. +version = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'otc-metadataReleaseNotesdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'otc-metadataReleaseNotes.tex', + 'otc-metadata Release Notes Documentation', + 'Open Telekom Cloud', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'otc-metadatarereleasenotes', + 'otc-metadata Release Notes Documentation', + ['Open Telekom Cloud'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'otc-metadata ReleaseNotes', + 'otc-metadata Release Notes Documentation', + 'Open Telekom Cloud Foundation', 'otc-metadataReleaseNotes', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 0000000..a040fa0 --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,8 @@ +============================================ + otc-metadata Release Notes +============================================ + +.. toctree:: + :maxdepth: 1 + + unreleased diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst new file mode 100644 index 0000000..cd22aab --- /dev/null +++ b/releasenotes/source/unreleased.rst @@ -0,0 +1,5 @@ +============================== + Current Series Release Notes +============================== + +.. release-notes:: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1d18dd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +pbr>=2.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ad735b1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,25 @@ +[metadata] +name = otc-metadata +summary = Metadata about OTC for Ecosystem +description_file = + README.rst +author = Open Telekom Cloud +home_page = https://open.telekom.cloud/ +python_requires = >=3.6 +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: Implementation :: CPython + +[files] +packages = + otc_metadata diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7f1bbe8 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..8a2bd6e --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,7 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +stestr>=2.0.0 # Apache-2.0 +testtools>=2.2.0 # MIT +flake8 diff --git a/tools-requirements.txt b/tools-requirements.txt new file mode 100644 index 0000000..1bddc55 --- /dev/null +++ b/tools-requirements.txt @@ -0,0 +1,7 @@ +GitPython +ruamel.yaml +requests +jinja2 +dirsync +cookiecutter +opensearch-py diff --git a/tools/bootstrap_repositories.py b/tools/bootstrap_repositories.py new file mode 100644 index 0000000..719ad38 --- /dev/null +++ b/tools/bootstrap_repositories.py @@ -0,0 +1,156 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +# import os +import pathlib +import subprocess +# import warnings + +# from git import exc +from git import Repo +# from git import SymbolicReference + +# from jinja2 import PackageLoader +# from jinja2 import Environment +# from jinja2 import select_autoescape + +from cookiecutter.main import cookiecutter + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def create_repo(repo, repo_dir, service): + repo_owner, repo_name = repo["repo"].split("/") + git_repo = None + + if repo_dir.exists(): + logging.debug(f"Repository {repo['repo']} is already existing") + return + + logging.info(f"Creating repository {repo_owner}/{repo_name}") + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + git_fqdn = "github.com" + + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.debug("Error") + pass + + if git_repo: + return + + if repo["type"] == "gitea": + subprocess.run( + args=[ + "tea", + "repo", + "create", + "--name", + repo_name, + "--owner", + repo_owner, + ] + ) + elif repo["type"] == "github": + subprocess.run( + args=[ + "gh", + "repo", + "create", + repo["repo"], + "--public", + ] + ) + cookiecutter( + template="https://github.com/opentelekomcloud/docs-cookiecutter", + output_dir=repo_dir.parent, + no_input=True, + extra_context=dict( + git_fqdn=git_fqdn, + repo_group=repo_owner, + repo_name=repo_name, + project_short_description=( + f"Documentation project for {service['service_title']} " + "service" + ), + overwrite_if_exists=True, + ), + ) + git_repo = Repo(repo_dir) + git_repo.create_remote("origin", repo_url) + git_repo.remotes.origin.fetch() + git_repo.git.push("--set-upstream", "origin", "main") + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] != args.target_environment: + continue + + checkout_exists = repo_dir.exists() + logging.debug(f"Repository {repo} exists {checkout_exists}") + repo_dir.parent.mkdir(parents=True, exist_ok=True) + if True: # not checkout_exists: + create_repo(repo, repo_dir, service) + + +def main(): + parser = argparse.ArgumentParser(description="Bootstrap repositories.") + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/convert_data.py b/tools/convert_data.py new file mode 100644 index 0000000..1d7fb3e --- /dev/null +++ b/tools/convert_data.py @@ -0,0 +1,22 @@ +import re + +import otc_metadata.services +from ruamel.yaml import YAML + +data = otc_metadata.services.Services() +new_data = data._service_data + +# services = data.service_dict + +for doc in new_data["documents"]: + hc_location = None + link = doc.get("link") + if link: + print(f"Parsing {link}") + # (p1, p2) = link.split("/") + doc["link"] = re.sub(r"/(.*)/(.*)/", r"/\2/\1/", link) + +_yaml = YAML() +_yaml.indent(mapping=2, sequence=4, offset=2) +with open("new.yaml", "w") as fd: + _yaml.dump(new_data, fd) diff --git a/tools/generate_doc_confpy.py b/tools/generate_doc_confpy.py new file mode 100644 index 0000000..2bd5d8f --- /dev/null +++ b/tools/generate_doc_confpy.py @@ -0,0 +1,477 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import pathlib +import requests +import subprocess + +from git import exc +from git import Repo + +from jinja2 import PackageLoader +from jinja2 import Environment +from jinja2 import select_autoescape + +import otc_metadata.services + +data = otc_metadata.services.Services() + +api_session = requests.Session() + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_to = None + repo_to = None + url_to = None + target_repo = None + git_fqdn = None + error_list = [] + + env = Environment( + loader=PackageLoader("otc_metadata"), autoescape=select_autoescape() + ) + conf_py_template = env.get_template("conf.py.j2") + tox_ini_template = env.get_template("tox.ini.j2") + zuul_yaml_template = env.get_template("zuul.yaml.j2") + index_sbv_template = env.get_template("index_sbv.rst.j2") + doc_requirements_template = env.get_template("doc_requirements.txt.j2") + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] == args.target_environment: + copy_to = repo_dir + else: + logging.debug(f"Skipping repository {repo}") + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + error_list.append({ + "error": f"Repository type {repo['type']} is not supported", + "repo": repo['repo'] + }) + continue + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + repo_dir.rmdir() + except Exception as e: + error_list.append({ + "error": e, + "repo": repo['repo'] + }) + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.error(f"Error cloning repository {repo_url}") + error_list.append({ + "error": f"Error cloning repository {repo_url}", + "repo": repo['repo'] + }) + continue + + if repo["environment"] == args.target_environment: + url_to = repo_url + repo_to = git_repo + target_repo = repo + break + + if not target_repo: + logging.info( + f"No repository service {service['service_title']}" + f"for environment {args.target_environment}" + ) + return + + branch_name = f"{args.branch_name}" + if args.branch_force: + logging.debug("Dropping current branch") + try: + repo_to.delete_head(branch_name, force=True) + repo_to.delete_head(f"refs/heads/{args.branch_name}", force=True) + except exc.GitCommandError as e: + print(e) + error_list.append({ + "error": e, + "repo": target_repo['repo'] + }) + pass + try: + new_branch = repo_to.create_head(branch_name, "main") + except Exception as e: + logging.warning(f"Skipping service {service} due to {e}") + error_list.append({ + "error": e, + "repo": target_repo['repo'] + }) + return + new_branch.checkout() + + service_docs = list(data.docs_by_service_type(service["service_type"])) + + for doc in service_docs: + logging.debug(f"Analyzing document {doc}") + + conf_py_path = pathlib.Path(copy_to, doc["rst_location"], "conf.py") + if not conf_py_path.exists(): + logging.info(f"Path for document {doc['title']} does not exist") + conf_py_path.parent.mkdir(parents=True, exist_ok=True) + context = dict( + repo_name=target_repo["repo"], + project=service["service_title"], + # pdf_name=doc["pdf_name"], + title=f"{service['service_title']} - {doc['title']}", + ) + if "pdf_name" in doc: + context["pdf_name"] = doc["pdf_name"] + if git_fqdn: + context["git_fqdn"] = git_fqdn + if target_repo.get("type") != "github": + context["git_type"] = target_repo["type"] + if (args.target_environment == "public" + and target_repo["repo"].split("/")[0] == "opentelekomcloud-docs-swiss"): + context["html_options"] = dict( + logo_url="https://docs.sc.otc.t-systems.com", + ) + elif (args.target_environment == "public"): + context["html_options"] = dict( + logo_url="https://docs.otc.t-systems.com", + ) + elif (args.target_environment == "internal" + and target_repo["repo"].split("/")[0] == "docs-swiss"): + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-swiss-int.otc-service.com", + ) + elif args.target_environment == "internal": + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-int.otc-service.com", + ) + context["doc_environment"] = args.target_environment + if doc['link']: + context["doc_link"] = doc['link'] + else: + context["doc_link"] = ( + '/' + + service['service_uri'] + + '/' + + doc['type'] + + '/' + ) + context["doc_title"] = doc['title'] + context["doc_type"] = doc['type'] + context["service_category"] = service['service_category'] + context["service_title"] = service['service_title'] + context["service_type"] = service['service_type'] + + conf_py_content = conf_py_template.render(**context) + with open(conf_py_path, "w", encoding="utf-8", newline="") as out: + logging.debug(f"Generating {conf_py_path} from template...") + out.write(conf_py_content) + + repo_to.index.add([doc["rst_location"]]) + + if args.update_sbv: + """Add or update service-based-view""" + copy_path = pathlib.Path(copy_to, 'doc', 'source') + context = dict( + repo_name=target_repo["repo"], + project=service["service_title"], + # pdf_name=doc["pdf_name"], + title=f"{service['service_title']} - Service Based View", + service_type=service["service_type"] + ) + context["service_category"] = service['service_category'] + context["service_title"] = service['service_title'] + if not copy_path.exists(): + logging.info("Path for sbv does not exist") + copy_path.mkdir(parents=True, exist_ok=True) + context["otc_sbv"] = True + if git_fqdn: + context["git_fqdn"] = git_fqdn + if target_repo.get("type") != "github": + context["git_type"] = target_repo["type"] + if (args.target_environment == "public" + and target_repo["repo"].split("/")[0] == "opentelekomcloud-docs-swiss"): + context["html_options"] = dict( + logo_url="https://docs.sc.otc.t-systems.com", + ) + elif (args.target_environment == "public"): + context["html_options"] = dict( + logo_url="https://docs.otc.t-systems.com", + ) + elif (args.target_environment == "internal" + and target_repo["repo"].split("/")[0] == "docs-swiss"): + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-swiss-int.otc-service.com", + ) + elif args.target_environment == "internal": + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-int.otc-service.com", + ) + context["environment"] = args.target_environment + sbv_title = (service["service_title"] + "\n" + + ('=' * len(service["service_title"]))) + context["sbv_title"] = sbv_title + conf_py_content = conf_py_template.render(**context) + index_sbv_content = index_sbv_template.render(**context) + with open( + pathlib.Path(copy_path, "conf.py"), + "w", + encoding="utf-8") as out: + out.write(conf_py_content) + repo_to.index.add(pathlib.Path(copy_path, "conf.py")) + + if (not args.overwrite_index_sbv + and pathlib.Path(copy_path, "index.rst").exists()): + logging.info("File index.rst for sbv exists. Skipping") + else: + with open( + pathlib.Path(copy_path, "index.rst"), + "w", + encoding="utf-8") as out: + out.write(index_sbv_content) + repo_to.index.add(pathlib.Path(copy_path, "index.rst")) + + placeholder_path = pathlib.Path(copy_path, "_static") + if not pathlib.Path(placeholder_path, "placeholder").exists(): + placeholder_path.mkdir(parents=True, exist_ok=True) + open(pathlib.Path(placeholder_path, "placeholder"), 'a').close() + repo_to.index.add(pathlib.Path(placeholder_path, "placeholder")) + + if args.update_tox: + """Update tox.ini""" + context = dict(docs=[]) + for doc in service_docs: + if doc["type"] == "dev": + doc["type"] = "dev-guide" + context["docs"].append(doc) + + context["target_environment"] = args.target_environment + context["service_type"] = service['service_type'] + + tox_ini_content = tox_ini_template.render(**context) + tox_ini_path = pathlib.Path(copy_to, "tox.ini") + doc_requirements_content = doc_requirements_template.render(**context) + doc_requirements_path = pathlib.Path( + copy_to, "doc", "requirements.txt" + ) + doc_requirements_path.parent.mkdir(parents=True, exist_ok=True) + with open(tox_ini_path, "w", encoding="utf-8", newline="") as out: + logging.debug(f"Generating {tox_ini_path} from template...") + out.write(tox_ini_content) + repo_to.index.add(["tox.ini"]) + with open( + doc_requirements_path, "w", encoding="utf-8", newline="" + ) as out: + logging.debug( + f"Generating {doc_requirements_path} from template..." + ) + out.write(doc_requirements_content) + repo_to.index.add(["doc/requirements.txt"]) + + if args.update_zuul: + """Update zuul.yaml""" + context = dict(docs=[]) + for doc in service_docs: + if doc["type"] == "dev": + doc["type"] = "dev-guide" + context["docs"].append(doc) + context["service_type"] = service['service_type'] + + zuul_yaml_content = zuul_yaml_template.render(**context) + zuul_yaml_path = pathlib.Path(copy_to, "zuul.yaml") + with open(zuul_yaml_path, "w", encoding="utf-8", newline="") as out: + logging.debug(f"Generating {zuul_yaml_path} from template...") + out.write(zuul_yaml_content) + repo_to.index.add(["zuul.yaml"]) + + if len(repo_to.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % service["service_type"] + ) + return + repo_to.index.commit( + args.commit_description + ) + push_args = ["--set-upstream", "origin", branch_name] + if args.force_push: + push_args.append("--force") + try: + repo_to.git.push(*push_args) + except Exception as e: + error_list.append({ + "error": e, + "repo": repo['repo'] + }) + if "github" in url_to: + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=False + ) + elif "gitea" in url_to and args.token: + open_pr( + args, + repo["repo"], + dict( + title="Update Docs configuration", + head=branch_name, + ), + ) + if len(error_list) != 0: + logging.error("The following errors have happened:") + logging.error(error_list) + + +def open_pr(args, repository, pr_data): + req = dict( + base=pr_data.get("base", "main"), + head=pr_data["head"], + ) + if "title" in pr_data: + req["title"] = pr_data["title"] + if "body" in pr_data: + req["body"] = pr_data["body"].replace("\\n", "\n") + if "assignees" in pr_data: + req["assignees"] = pr_data["assignees"] + if "labels" in pr_data: + req["labels"] = pr_data["labels"] + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/pulls", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + # print(f"Going to open PR with title {pr_data['title']} in {repository}") + + +def main(): + parser = argparse.ArgumentParser( + description="Update conf.py file in repositories." + ) + parser.add_argument( + "--target-environment", + required=True, + choices=["internal", "public"], + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--update-tox", action="store_true", help="Whether to update tox.ini." + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="confpy", + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--branch-force", + action="store_true", + help="Whether to force branch recreation.", + ) + parser.add_argument("--token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + parser.add_argument( + "--update-sbv", + action="store_true", + help="Whether to update service-based-view" + ) + parser.add_argument( + "--update-zuul", + action="store_true", + help="Whether to update zuul.yaml" + ) + parser.add_argument( + "--overwrite-index-sbv", + action="store_true", + help=("Whether to overwrite index.rst for service-based-view." + + "\nCan only be used if --update-sbv is also specified") + ) + parser.add_argument( + "--force-push", + action="store_true", + help="Whether to force push the commit" + ) + parser.add_argument( + "--commit-description", + default=( + "Update tox.ini && conf.py file\n\n" + "Performed-by: gitea/infra/otc-metadata/" + "tools/generate_doc_confpy.py" + ), + help="Commit description for the commit", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.overwrite_index_sbv and not args.update_sbv: + logging.error( + "Cannot overwrite index.rst for service-based-view" + + " without updating service-based-view" + ) + exit(1) + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + if args.token: + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_doc_gitcontrol_repos.py b/tools/generate_doc_gitcontrol_repos.py new file mode 100644 index 0000000..b8e344f --- /dev/null +++ b/tools/generate_doc_gitcontrol_repos.py @@ -0,0 +1,186 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import copy +import logging +import yaml + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_services(args, services): + """Process services""" + gitea_bp = dict( + branch_name="main", + enable_approvals_whitelist=True, + approvals_whitelist_teams=["docs-infra-team"], + block_on_rejected_reviews=True, + dismiss_stale_approvals=True, + enable_push=False, + enable_status_check=True, + status_check_contexts=["gl/check"], + enable_merge_whitelist=True, + merge_whitelist_usernames=["zuul"], + required_approvals=1, + ) + gitea_repo_template = dict( + default_branch="main", + description="Open Telekom Cloud Service docs", + homepage=None, + archived=False, + has_issues=True, + has_projects=False, + has_wiki=False, + default_delete_branch_after_merge=True, + allow_merge_commit=False, + allow_squash_merge=True, + allow_rebase_merge=False, + default_merge_style="squash", + branch_protections=[], + ) + github_bp = dict( + branch_name="main", + enable_approvals_whitelist=True, + approvals_whitelist_teams=["docs-infra-team"], + block_on_rejected_reviews=True, + dismiss_stale_approvals=True, + enable_push=False, + status_check_contexts=["gl/check"], + enable_merge_whitelist=True, + merge_whitelist_usernames=["zuul"], + ) + github_repo_template = dict( + default_branch="main", + description="Open Telekom Cloud Service docs", + homepage=None, + archived=False, + has_issues=True, + has_projects=False, + has_wiki=False, + delete_branch_on_merge=True, + allow_merge_commit=False, + allow_squash_merge=True, + allow_rebase_merge=False, + allow_update_branch=True, + branch_protections=[], + ) + + for service in services: + logging.debug(f"Processing service {service}") + config = None + repo_name = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + if repo["environment"] != args.target_environment: + continue + + if repo["type"] == "gitea": + teams = [] + branch_protections_main = copy.deepcopy(gitea_bp) + if "teams" in repo: + teams_def = repo["teams"] + if "teams" in service: + teams_def = service["teams"] + if teams_def: + for team in teams_def: + branch_protections_main[ + "approvals_whitelist_teams" + ].append(team["name"]) + teams.append(team["name"]) + data = copy.deepcopy(gitea_repo_template) + data["description"] = ( + f"Open Telekom Cloud {service['service_title']} " + f"Service docs" + ) + data["branch_protections"].append(branch_protections_main) + data["teams"] = teams + repo_name = repo["repo"].split("/")[1] + config = {repo_name: data} + + elif repo["type"] == "github": + teams = [] + branch_protections_main = copy.deepcopy(github_bp) + if "teams" in repo: + teams_def = repo["teams"] + if "teams" in service: + teams_def = service["teams"] + if teams_def: + for team in teams_def: + teams.append( + { + "slug": team["name"], + "permission": "push" + if team["permission"] == "write" + else "pull", + } + ) + data = copy.deepcopy(github_repo_template) + data["description"] = ( + f"Open Telekom Cloud {service['service_title']} " f"docs" + ) + data["branch_protections"].append( + {"branch": "main", "template": "zuul"} + ) + data["teams"] = teams + repo_name = repo["repo"].split("/")[1] + config = {repo_name: data} + + else: + logging.error( + "Repository type %s is not supported", repo["type"] + ) + exit(1) + + if data: + with open( + f"{args.work_dir}/{repo_name}.yml", + "w", + encoding="utf-8", + newline="", + ) as out: + logging.debug( + "Generating %s config from template...", + service["service_type"], + ) + yaml.dump(config, out) + + +def main(): + parser = argparse.ArgumentParser( + description="Update conf.py file in repositories." + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + + process_services(args, data.all_services) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_docexports_data.py b/tools/generate_docexports_data.py new file mode 100644 index 0000000..8d583fd --- /dev/null +++ b/tools/generate_docexports_data.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys + +from ruamel.yaml import YAML + +import otc_metadata.services + + +def main(): + data = otc_metadata.services.Services() + data._sort_data() + + docs = data.docs_html_by_category("internal") + + # Filter out documents with "disable_import": True + for category, services in docs['categories'].items(): + for service in services: + filtered_docs = [] + + for doc in service['docs']: + # Check if the document doesnt have 'disable_import' on True + if not doc.get('disable_import'): + filtered_docs.append(doc) + + service['docs'] = filtered_docs + + _yaml = YAML() + _yaml.indent(mapping=2, sequence=4, offset=2) + sys.stdout.write( + "# Auto-generated by otc_metadata.generate_docexports.data\n" + ) + _yaml.dump(docs, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/tools/index_metadata.py b/tools/index_metadata.py new file mode 100644 index 0000000..25252dc --- /dev/null +++ b/tools/index_metadata.py @@ -0,0 +1,158 @@ +import otc_metadata +import argparse +import logging +from opensearchpy import OpenSearch + + +metadata = otc_metadata.Services() + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Create Index data for search inside OpenSearch" + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + '--delete-index', + action='store_true', + help='Option deletes old index with the same name and creates new ' + 'one.' + ) + parser.add_argument( + '--all-doc-types', + action='store_true', + help='Upload all doc-types instead of only umn, api-ref and dev' + ) + parser.add_argument( + '--debug', + action='store_true', + help='Enable Debug mode' + ) + parser.add_argument( + '--hosts', + metavar='<host:port>', + nargs='+', + default=['localhost:9200'], + help='Provide one or multiple host:port values ' + 'separated by space for multiple hosts.\n' + 'Default: localhost:9200' + ) + parser.add_argument( + '--index', + metavar='<index>', + default='test-index', + help="OpenSearch / ElasticSearch index name.\n" + 'Default: test-index' + ) + parser.add_argument( + '--username', + metavar='<username>', + required=True, + help='Username for the connection.' + ) + parser.add_argument( + '--password', + metavar='<password>', + required=True, + help='Password for the connection.' + ) + + args = parser.parse_args() + return args + + +def main(): + + args = parse_args() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + + logging.debug("Obtaining data from otc_metadata") + data = getData( + environment=args.target_environment, + all_doc_types=args.all_doc_types + ) + + sorted_services = sortData(data['services'], sort_key='service_title') + sorted_data = {'services': sorted_services, 'docs': data['docs']} + + logging.debug("Indexing data into OpenSearch") + indexData( + deleteIndex=args.delete_index, + hosts=args.hosts, + index=args.index, + username=args.username, + password=args.password, + data=sorted_data + ) + + +def filter_docs(metadata): + allowed_types = ["umn", "api-ref", "dev"] + metadata['docs'] = [doc for doc in metadata['docs'] + if doc['type'] in allowed_types] + return metadata + + +def getData(environment, all_doc_types): + metadatadata = metadata.service_types_with_doc_types( + environment=environment + ) + final_data = metadatadata + if not all_doc_types: + final_data = filter_docs(metadatadata) + return final_data + + +def sortData(data, sort_key): + return sorted(data, key=lambda x: x[sort_key]) + + +def indexData(deleteIndex, hosts, index, username, password, data): + hosts = generate_os_host_list(hosts) + client = OpenSearch( + hosts=hosts, + http_compress=True, + http_auth=(username, password), + use_ssl=True, + verify_certs=True, + ssl_assert_hostname=False, + ssl_show_warn=False + ) + + if deleteIndex is True: + logging.debug("Deleting Index") + delete_index(client, index) + + logging.debug("Started creating Index") + create_index(client, index, data) + logging.debug("Finished creating Index") + + +def generate_os_host_list(hosts): + host_list = [] + for host in hosts: + raw_host = host.split(':') + if len(raw_host) != 2: + raise Exception('--hosts parameter does not match the following ' + 'format: hostname:port') + json_host = {'host': raw_host[0], 'port': int(raw_host[1])} + host_list.append(json_host) + return host_list + + +def create_index(client, index, data): + client.indices.create(index=index) + return client.index(index=index, body=data) + + +def delete_index(client, index): + return client.indices.delete(index=index, ignore=[400, 404]) + + +main() diff --git a/tools/open_doc_issue.py b/tools/open_doc_issue.py new file mode 100644 index 0000000..bf4b34b --- /dev/null +++ b/tools/open_doc_issue.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import argparse +# import re +import requests +# import sys + +import otc_metadata.services + +api_session = requests.Session() + + +def open_issue(args, repository, issue_data): + req = dict( + title=issue_data["title"], body=issue_data["body"].replace("\\n", "\n") + ) + if "assignees" in issue_data: + req["assignees"] = issue_data["assignees"] + if "labels" in issue_data: + req["labels"] = issue_data["labels"] + print(req) + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/issues", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + print( + f"Going to open issue with title {issue_data['title']} in {repository}" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Open Issue for every document." + ) + parser.add_argument("token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + parser.add_argument("--environment", help="Environment for the repository") + parser.add_argument("--title", required=True, help="Issue title") + parser.add_argument("--body", required=True, help="Issue body") + parser.add_argument( + "--repo", + help="Repository to report issue in (instead of doc repository).", + ) + parser.add_argument( + "--assignee", + help="Issue assignee to use instead of document service assignees.", + ) + parser.add_argument( + "--labels", + help="Issue labels to use (comma separated list of label IDs).", + ) + args = parser.parse_args() + data = otc_metadata.services.Services() + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for doc in data.all_docs_full(environment=args.environment): + issue_data = dict( + title=args.title.format(**doc), + body=args.body.format(**doc), + repository=doc["repository"], + ) + if "assignees" in doc: + issue_data["assignees"] = doc["assignees"] + if args.assignee: + issue_data["assignees"] = [args.assignee] + if args.labels: + issue_data["labels"] = [int(x) for x in args.labels.split(",")] + open_issue(args, args.repo or doc["repository"], issue_data) + + +if __name__ == "__main__": + main() diff --git a/tools/sort_metadata.py b/tools/sort_metadata.py new file mode 100644 index 0000000..caeae59 --- /dev/null +++ b/tools/sort_metadata.py @@ -0,0 +1,9 @@ +# import copy + +import otc_metadata.services +# from ruamel.yaml import YAML + +data = otc_metadata.services.Services() + +data._sort_data() +data._rewrite_data() diff --git a/tools/split_metadata.py b/tools/split_metadata.py new file mode 100644 index 0000000..1db2497 --- /dev/null +++ b/tools/split_metadata.py @@ -0,0 +1,41 @@ +# import copy + +from pathlib import Path +import yaml + +import otc_metadata + +data = otc_metadata.services.Services() + +for item in data.all_docs: + with open( + Path( + otc_metadata.data.DATA_DIR, + "documents", + f"{item['service_type']}-{item['type']}.yaml", + ), + "w", + ) as fp: + yaml.dump(item, fp, explicit_start=True) + +for item in data.all_services: + with open( + Path( + otc_metadata.data.DATA_DIR, + "services", + f"{item['service_type']}.yaml", + ), + "w", + ) as fp: + yaml.dump(item, fp, explicit_start=True) + +for item in data.service_categories: + with open( + Path( + otc_metadata.data.DATA_DIR, + "service_categories", + f"{item['name']}.yaml", + ), + "w", + ) as fp: + yaml.dump(item, fp, explicit_start=True) diff --git a/tools/sync_doc_repo.py b/tools/sync_doc_repo.py new file mode 100644 index 0000000..92f03a9 --- /dev/null +++ b/tools/sync_doc_repo.py @@ -0,0 +1,201 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +import os +import pathlib +import subprocess +import warnings +from dirsync import sync + +from git import Repo +from git import SymbolicReference + + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_repositories(args, service): + """Checkout repositories""" + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_from = None + copy_to = None + repo_to = None + url_from = None + url_to = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] == args.source_environment: + copy_from = repo_dir + elif repo["environment"] == args.target_environment: + copy_to = repo_dir + else: + continue + + checkout_exists = repo_dir.exists() + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + if not checkout_exists: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + else: + logging.debug("Checkout already exists") + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + + if repo["environment"] == args.source_environment: + url_from = repo_url + elif repo["environment"] == args.target_environment: + url_to = repo_url + repo_to = git_repo + logging.debug(f"Synchronizing from={url_from}, to={url_to}") + + if not copy_from or not copy_to: + logging.warn( + "Not synchronizing documents of the service. " + "Target or source not known." + ) + + for doc in data.docs_by_service_type(service["service_type"]): + logging.debug(f"Analyzing document {doc}") + if args.document_type and doc.get("type") != args.document_type: + logging.info( + f"Skipping synchronizing {doc['title']} " + f"due to the doc-type filter." + ) + continue + # if doc.get("environment"): + # logging.info( + # f"Skipping synchronizing {doc['title']} " + # f"since it is environment bound." + # ) + # continue + + branch_name = f"{args.branch_name}#{doc['type']}" + remote_ref = SymbolicReference.create( + repo_to, "refs/remotes/origin/%s" % branch_name + ) + new_branch = repo_to.create_head(branch_name, "main") + remote_ref = repo_to.remotes[0].refs[branch_name] # get correct type + new_branch.set_tracking_branch(remote_ref) + new_branch.checkout() + + source_path = pathlib.Path(copy_from, doc["rst_location"]) + target_path = pathlib.Path(copy_to, doc["rst_location"]) + sync( + source_path, + target_path, + 'sync', + purge=True, + create=True, + content=True, + ignore=['conf.py'] + ) + repo_to.index.add([doc["rst_location"]]) + + for obj in repo_to.index.diff(None).iter_change_type('D'): + repo_to.index.remove([obj.b_path]) + if len(repo_to.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug("No changes.") + continue + repo_to.index.commit( + ( + f"Synchronize {doc['title']}\n\n" + f"Overwriting document\n" + f"from: {url_from}\n" + f"to: {url_to}\n\n" + "Performed-by: gitea/infra/otc-metadata/tools/sync_doc_repo.py" + ) + ) + repo_to.remotes.origin.push(new_branch) + if "github" in url_to and args.open_pr_gh: + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Synchronize document between environments." + ) + parser.add_argument( + "--source-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--service-type", + required=True, + help="Service to which document(s) belongs to", + ) + parser.add_argument( + "--document-type", + help=( + "Type of the document to synchronize. " + "All will be synchronized if not set." + ), + ) + parser.add_argument( + "--branch-name", + required=True, + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--open-pr-gh", + action="store_true", + help="Open Pull Request using `gh` utility (need to be present).", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + service = data.service_dict.get(args.service_type) + if not service: + warnings.warn(f"Service {args.service_type} was not found in metadata") + os.exit(1) + + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/update_zuul_project_configs.py b/tools/update_zuul_project_configs.py new file mode 100644 index 0000000..4fe5374 --- /dev/null +++ b/tools/update_zuul_project_configs.py @@ -0,0 +1,294 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +# import os +import pathlib +import requests +import subprocess +# import warnings + +from git import exc +from git import Repo +# from git import SymbolicReference + +from ruamel.yaml import CommentedMap +from ruamel.yaml import YAML + +import otc_metadata.services + +data = otc_metadata.services.Services() + +yaml = YAML() + +api_session = requests.Session() + + +def load_zuul_config(workdir): + for path in [".zuul.yaml", "zuul.yaml"]: + test_path = pathlib.Path(workdir, path) + if test_path.exists(): + with open(test_path, "r") as f: + zuul_config = yaml.load(f) + return (zuul_config, test_path) + + +def open_pr(args, repository, pr_data): + req = dict( + title=pr_data["title"], + body=pr_data["body"].replace("\\n", "\n"), + base=pr_data.get("base", "main"), + head=pr_data["head"], + ) + if "assignees" in pr_data: + req["assignees"] = pr_data["assignees"] + if "labels" in pr_data: + req["labels"] = pr_data["labels"] + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/pulls", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + print(f"Going to open PR with title {pr_data['title']} in {repository}") + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_to = None + # repo_to = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] != args.environment: + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.update() + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + repo_dir.rmdir() + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.error(f"Error cloning repository {repo_url}") + return + + branch_name = f"{args.branch_name}" + if args.branch_force: + logging.debug("Dropping current branch") + try: + git_repo.delete_head(branch_name, force=True) + except exc.GitCommandError: + pass + try: + new_branch = git_repo.create_head(branch_name, "main") + except Exception as ex: + logging.warning(f"Skipping service {service} due to {ex}") + return + new_branch.checkout() + + (zuul_config, zuul_file_name) = load_zuul_config(repo_dir) + zuul_templates = None + zuul_jobs = dict() + zuul_new_jobs = list() + zuul_config_updated = False + for item in zuul_config: + if "project" in item.keys(): + project = item["project"] + zuul_templates = project.setdefault("templates", []) + if not zuul_templates: + zuul_templates = [] + elif "job" in item.keys(): + job = item["job"] + zuul_jobs[job["name"]] = job + print(f"Existing jobs {zuul_jobs}") + if "helpcenter-base-jobs" not in zuul_templates: + zuul_templates.append("helpcenter-base-jobs") + zuul_config_updated = True + + job_suffix = ( + "-hc-int-jobs" if args.environment == "internal" else "-hc-jobs" + ) + for doc in data.docs_by_service_type(service["service_type"]): + logging.debug(f"Analyzing document {doc}") + if not doc.get("type"): + continue + if doc["type"] == "dev": + doc_type = "dev-guide" + else: + doc_type = doc["type"] + template_name = f"{doc_type}{job_suffix}" + if doc_type in ["api-ref", "umn", "dev-guide"]: + if template_name not in zuul_templates: + zuul_templates.append(template_name) + else: + job_name = f"build-otc-{doc['service_type']}-{doc_type}" + if job_name not in zuul_jobs: + zuul_config_updated = True + zuul_new_jobs.append( + dict( + job=dict( + name=job_name, + parent="otc-tox-docs", + description=( + f"Build {doc_type} document using tox" + ), + files=[ + f"^{doc['rst_location']}/.*" + ], + vars=dict( + tox_envlist=doc_type + ) + ) + ) + ) + + if zuul_config_updated: + for new_job in zuul_new_jobs: + zuul_config.insert(0, new_job) + + for item in zuul_config: + if "project" in item.keys(): + project = item["project"] + project["templates"] = zuul_templates + # Ensure new jobs are in check + if len(zuul_new_jobs) > 0: + project.setdefault( + "check", + CommentedMap(jobs=[]) + ) + project["check"].yaml_set_comment_before_after_key( + key="jobs", + indent=6, + before=( + "Separate documents are rendered in check, " + "while published through regular " + "otc-tox-docs job part of the basic template" + ) + ) + project["check"]["jobs"].extend( + [x["job"]["name"] for x in zuul_new_jobs]) + + # yaml.indent(offset=2, sequence=2) + with open(zuul_file_name, "w") as f: + yaml.dump(zuul_config, f) + git_repo.index.add([zuul_file_name.name]) + if len(git_repo.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % + service["service_type"]) + return + + git_repo.index.commit( + ( + "Update zuul.yaml file\n\n" + "Performed-by: gitea/infra/otc-metadata" + "/tools/update_zuul_project_config.py" + ) + ) + push_args = ["--set-upstream", "origin", branch_name] + if args.branch_force: + push_args.append("--force") + + git_repo.git.push(*push_args) + if repo["type"] == "github": + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True + ) + elif repo["type"] == "gitea": + open_pr( + args, + repo["repo"], + dict( + title="Update zuul config", + body="Update zuul config templates", + head=branch_name, + ), + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Update zuul.yaml file in repositories." + ) + parser.add_argument( + "--environment", + required=True, + help="Repository Environment", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="zuul", + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--branch-force", + action="store_true", + help="Whether to force branch recreation.", + ) + parser.add_argument("--token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + if args.token: + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..74aa87e --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +minversion = 3.2.0 +envlist = py3,pep8 +ignore_basepython_conflict = true + +[testenv] +basepython = python3 +usedevelop = True +setenv = + PYTHONWARNINGS=default::DeprecationWarning +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run {posargs} + +[testenv:pep8] +commands = flake8 {posargs} + +[testenv:venv] +commands = {posargs} + +[flake8] +# E123, E125 skipped as they are invalid PEP-8. +show-source = True +ignore = E123,E125,W503,E501 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build diff --git a/zuul.yaml b/zuul.yaml new file mode 100644 index 0000000..3a1ad47 --- /dev/null +++ b/zuul.yaml @@ -0,0 +1,12 @@ +--- +- project: + merge-mode: squash-merge + default-branch: main + check: + jobs: + - otc-tox-pep8 + - otc-tox-py39 + gate: + jobs: + - otc-tox-pep8 + - otc-tox-py39