From 408b2ece8908fcb9e03991df2d45c832c33f0c8c Mon Sep 17 00:00:00 2001 From: Vladimir Hasko Date: Wed, 17 May 2023 14:02:14 +0000 Subject: [PATCH] adding Swisscloud content metadata --- .coveragerc | 6 + .gitignore | 61 ++ .gitreview | 4 + .mailmap | 3 + .stestr.conf | 3 + CONTRIBUTING.rst | 3 + HACKING.rst | 4 + LICENSE | 176 ++++ README.rst | 54 + doc/requirements.txt | 4 + doc/source/admin/index.rst | 5 + doc/source/cli/index.rst | 5 + doc/source/conf.py | 76 ++ doc/source/configuration/index.rst | 5 + doc/source/contributor/contributing.rst | 5 + doc/source/contributor/index.rst | 9 + doc/source/index.rst | 30 + doc/source/install/index.rst | 14 + doc/source/install/install.rst | 13 + doc/source/library/index.rst | 7 + doc/source/readme.rst | 1 + doc/source/reference/index.rst | 5 + doc/source/user/index.rst | 5 + otc_metadata/__init__.py | 35 + otc_metadata/data/__init__.py | 41 + otc_metadata/data/services.yaml | 985 ++++++++++++++++++ otc_metadata/docs.py | 38 + otc_metadata/services.py | 230 ++++ otc_metadata/templates/conf.py.j2 | 122 +++ .../templates/doc_requirements.txt.j2 | 7 + otc_metadata/templates/index_sbv.rst.j2 | 7 + otc_metadata/templates/tox.ini.j2 | 125 +++ otc_metadata/tests/__init__.py | 0 otc_metadata/tests/test_otc_metadata.py | 65 ++ releasenotes/notes/.placeholder | 0 releasenotes/source/_static/.placeholder | 0 releasenotes/source/_templates/.placeholder | 0 releasenotes/source/conf.py | 271 +++++ releasenotes/source/index.rst | 8 + releasenotes/source/unreleased.rst | 5 + requirements.txt | 5 + setup.cfg | 25 + setup.py | 18 + test-requirements.txt | 7 + tools-requirements.txt | 6 + tools/bootstrap_repositories.py | 156 +++ tools/convert_data.py | 22 + tools/generate_doc_confpy.py | 363 +++++++ tools/generate_doc_gitcontrol_repos.py | 184 ++++ tools/generate_docexports_data.py | 23 + tools/open_doc_issue.py | 73 ++ tools/service_list | 50 + tools/sort_metadata.py | 9 + tools/sync_doc_repo.py | 201 ++++ tools/update_zuul_project_configs.py | 294 ++++++ tox.ini | 25 + zuul.yaml | 12 + 57 files changed, 3910 insertions(+) create mode 100644 .coveragerc create mode 100644 .gitignore create mode 100644 .gitreview create mode 100644 .mailmap create mode 100644 .stestr.conf create mode 100644 CONTRIBUTING.rst create mode 100644 HACKING.rst create mode 100644 LICENSE create mode 100644 README.rst create mode 100644 doc/requirements.txt create mode 100644 doc/source/admin/index.rst create mode 100644 doc/source/cli/index.rst create mode 100755 doc/source/conf.py create mode 100644 doc/source/configuration/index.rst create mode 100644 doc/source/contributor/contributing.rst create mode 100644 doc/source/contributor/index.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/install/index.rst create mode 100644 doc/source/install/install.rst create mode 100644 doc/source/library/index.rst create mode 100644 doc/source/readme.rst create mode 100644 doc/source/reference/index.rst create mode 100644 doc/source/user/index.rst create mode 100644 otc_metadata/__init__.py create mode 100644 otc_metadata/data/__init__.py create mode 100644 otc_metadata/data/services.yaml create mode 100644 otc_metadata/docs.py create mode 100644 otc_metadata/services.py create mode 100644 otc_metadata/templates/conf.py.j2 create mode 100644 otc_metadata/templates/doc_requirements.txt.j2 create mode 100644 otc_metadata/templates/index_sbv.rst.j2 create mode 100644 otc_metadata/templates/tox.ini.j2 create mode 100644 otc_metadata/tests/__init__.py create mode 100644 otc_metadata/tests/test_otc_metadata.py create mode 100644 releasenotes/notes/.placeholder create mode 100644 releasenotes/source/_static/.placeholder create mode 100644 releasenotes/source/_templates/.placeholder create mode 100644 releasenotes/source/conf.py create mode 100644 releasenotes/source/index.rst create mode 100644 releasenotes/source/unreleased.rst create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100644 tools-requirements.txt create mode 100644 tools/bootstrap_repositories.py create mode 100644 tools/convert_data.py create mode 100644 tools/generate_doc_confpy.py create mode 100644 tools/generate_doc_gitcontrol_repos.py create mode 100644 tools/generate_docexports_data.py create mode 100644 tools/open_doc_issue.py create mode 100644 tools/service_list create mode 100644 tools/sort_metadata.py create mode 100644 tools/sync_doc_repo.py create mode 100644 tools/update_zuul_project_configs.py create mode 100644 tox.ini create mode 100644 zuul.yaml diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..485dd01 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True +source = otc-metadata-swiss + +[report] +ignore_errors = True diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4cbb844 --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +# Add patterns in here to exclude files created by tools integrated with this +# repository, such as test frameworks from the project's recommended workflow, +# rendered documentation and package builds. +# +# Don't add patterns to exclude files created by preferred personal tools +# (editors, IDEs, your operating system itself even). These should instead be +# maintained outside the repository, for example in a ~/.gitignore file added +# with: +# +# git config --global core.excludesfile '~/.gitignore' + +# Bytecompiled Python +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg* +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +cover/ +.coverage* +!.coveragerc +.tox +nosetests.xml +.testrepository +.stestr +.venv + +# Translations +*.mo + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Files created by releasenotes build +releasenotes/build diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..f497f2b --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.opendev.org +port=29418 +project=infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git.git diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..516ae6f --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# +# diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..6b9459b --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./otc_metadata/tests/ +top_dir=./ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..30439f0 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,3 @@ +The source repository for this project can be found at: + + https://github.com/infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000..6c09a78 --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,4 @@ +ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git Style Commandments +=============================================== + +Read the OpenStack Style Commandments https://docs.openstack.org/hacking/latest/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..68c771a --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..1336a36 --- /dev/null +++ b/README.rst @@ -0,0 +1,54 @@ +============ +otc-metadata +============ + +Link: ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git + + +Metadata about OTC for Ecosystem in Swisscloud environment + +Please fill here a long description which must be at least 3 lines wrapped on +80 cols, so that distribution package maintainers can use it in their packages. +Note that this is a hard requirement. + +* Free software: Apache license +* Documentation: https://docs.otc.t-systems.com/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git +* Source: https://github.com/infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git + +Features +======== + +* TODO + +Overview: service.yaml +====================== + +The :code:`service.yaml` file contains all data about services, service +categories and the related documents of each service. The file is +used as a base for several internal and external applications or +websites like the Helpcenter 3.0 where the information about the document +repositories and its properties are stored. + +File structure +-------------- + +The file is based on the yaml-file format and has three main sections +which can be compared with database tables in a relational database. + +* documents: contains the information about every single document and its type + like umn, api-ref etc. + +* service category: contains the keyword and title of the service category + +* services: contains the repository information about the internal (Gitea) and + external location (GitHub) and all the necessary parameters of the service itself + +These sections, or better "tables" have +their own keys and foreign keys so that the tables are linked together and +the related information can be fetched. +For the :code:`services` table +the key is :code:`service_type` which has the foreign key in the +:code:`documents` table. So a service can have multiple documents and each +document can only be linked to one service. +The key :code:`service_category` table is :code:`name` of the service category +which is then used in the :code:`services` table as foreign key. diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..23b871f --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,4 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +otcdocstheme>=1.0.0 # Apache-2.0 +# releasenotes +reno>=3.1.0 # Apache-2.0 diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst new file mode 100644 index 0000000..1718fde --- /dev/null +++ b/doc/source/admin/index.rst @@ -0,0 +1,5 @@ +==================== +Administrators guide +==================== + +Administrators guide of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst new file mode 100644 index 0000000..7b843a2 --- /dev/null +++ b/doc/source/cli/index.rst @@ -0,0 +1,5 @@ +================================ +Command line interface reference +================================ + +CLI reference of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100755 index 0000000..9a75852 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'otcdocstheme', + #'sphinx.ext.intersphinx', +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git' +copyright = '2022, Open Telekom Cloud Developers' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] +html_theme = 'otcdocs' + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + '%s Documentation' % project, + 'Open Telekom Cloud Developers', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst new file mode 100644 index 0000000..5394ea5 --- /dev/null +++ b/doc/source/configuration/index.rst @@ -0,0 +1,5 @@ +============= +Configuration +============= + +Configuration of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/contributor/contributing.rst b/doc/source/contributor/contributing.rst new file mode 100644 index 0000000..2e91325 --- /dev/null +++ b/doc/source/contributor/contributing.rst @@ -0,0 +1,5 @@ +============================ +So You Want to Contribute... +============================ + +TODO diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 0000000..036e449 --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1,9 @@ +=========================== + Contributor Documentation +=========================== + +.. toctree:: + :maxdepth: 2 + + contributing + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..6a96f30 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,30 @@ +.. ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +============================================ +Welcome to the documentation of otc-metadata +============================================ + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + install/index + library/index + contributor/index + configuration/index + cli/index + user/index + admin/index + reference/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst new file mode 100644 index 0000000..6d7dc44 --- /dev/null +++ b/doc/source/install/index.rst @@ -0,0 +1,14 @@ +================================================ + {{cookiecutter.module_name}} installation guide +================================================ + +.. toctree:: + :maxdepth: 2 + + install.rst + +The otc-metadata provides... + +This chapter assumes a working setup of OpenStack following the +`OpenStack Installation Tutorial +`_. diff --git a/doc/source/install/install.rst b/doc/source/install/install.rst new file mode 100644 index 0000000..3d5c5a7 --- /dev/null +++ b/doc/source/install/install.rst @@ -0,0 +1,13 @@ +.. _install: + +Install and configure +~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the +otc-metadata. + +This section assumes that you already have a working OpenStack +environment with at least the following components installed: +.. (add the appropriate services here and further notes) + +Note that installation and configuration vary by distribution. diff --git a/doc/source/library/index.rst b/doc/source/library/index.rst new file mode 100644 index 0000000..e884b9b --- /dev/null +++ b/doc/source/library/index.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git in a project:: + + import otc-metadata diff --git a/doc/source/readme.rst b/doc/source/readme.rst new file mode 100644 index 0000000..a6210d3 --- /dev/null +++ b/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst new file mode 100644 index 0000000..1d57317 --- /dev/null +++ b/doc/source/reference/index.rst @@ -0,0 +1,5 @@ +========== +References +========== + +References of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..efbc805 --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,5 @@ +=========== +Users guide +=========== + +Users guide of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/otc_metadata/__init__.py b/otc_metadata/__init__.py new file mode 100644 index 0000000..56b98c7 --- /dev/null +++ b/otc_metadata/__init__.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +__all__ = ['__version__', 'Docs'] + +import pbr.version + +from otc_metadata.services import Services # flake8: noqa + +__version__ = pbr.version.VersionInfo('otc-metadata').version_string() +_service_manager = None + + +def get_service_data(*args, **kwargs): + """Return singleton instance of the Services object. + Parameters are all passed through to the + :class:`~otc_metadata.services.Services` constructor. + .. note:: + Only one singleton is kept, so if instances with different parameter + values are desired, directly calling the constructor is necessary. + :returns: Singleton instance of + :class:`~otc_metadata.services.Services` + """ + global _service_manager + if not _service_manager: + _service_manager = Services(*args, **kwargs) + return _service_manager diff --git a/otc_metadata/data/__init__.py b/otc_metadata/data/__init__.py new file mode 100644 index 0000000..2102a7e --- /dev/null +++ b/otc_metadata/data/__init__.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml + +__all__ = ['read_data'] + +DATA_DIR = os.path.dirname(__file__) + + +def read_data(filename): + """Return data that is shipped inside the Python package. + + """ + + filepath = os.path.join(DATA_DIR, filename) + with open(filepath, 'r') as fd: + return yaml.safe_load(fd) + + +def rewrite_data(filename, data): + """Rewrites data formatting it + + """ + from ruamel.yaml import YAML + _yaml = YAML() + _yaml.indent(mapping=2, sequence=4, offset=2) + filepath = os.path.join(DATA_DIR, filename) + with open(filepath, 'w') as fd: + _yaml.dump(data, fd) diff --git a/otc_metadata/data/services.yaml b/otc_metadata/data/services.yaml new file mode 100644 index 0000000..0323254 --- /dev/null +++ b/otc_metadata/data/services.yaml @@ -0,0 +1,985 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please consult with otc_metadata.services.Services:_sort_data for expected +# sort order. When unsure perform tools/sort_metadata.py for getting it sorted. +--- +documents: + - hc_location: api/aom + html_location: docs/aom/api-ref + link: /application-operations-management/api-ref/ + pdf_name: aom-api-ref + rst_location: api-ref/source + service_type: aom + title: API Reference + type: api-ref + - hc_location: usermanual/aom + html_location: docs/aom/umn + link: /application-operations-management/umn/ + pdf_name: aom-umn + rst_location: umn/source + service_type: aom + title: User Guide + type: umn + - hc_location: api/as + html_location: docs/as/api-ref + link: /auto-scaling/api-ref/ + pdf_name: as-api-ref + rst_location: api-ref/source + service_type: as + title: API Reference + type: api-ref + - hc_location: devg/as + html_location: docs/as/dev + link: /auto-scaling/dev-guide/ + pdf_name: as-dev-guide + rst_location: dev_guide/source + service_type: as + title: Developer Guide + type: dev + - hc_location: usermanual/as + html_location: docs/as/umn + link: /auto-scaling/umn/ + pdf_name: as-umn + rst_location: umn/source + service_type: as + title: User Guide + type: umn + - hc_location: api/cbr + html_location: docs/cbr/api-ref + link: /cloud-backup-recovery/api-ref/ + pdf_name: cbr-api-ref + rst_location: api-ref/source + service_type: cbr + title: API Reference + type: api-ref + - hc_location: usermanual/cbr + html_location: docs/cbr/umn + link: /cloud-backup-recovery/umn/ + pdf_name: cbr-umn + rst_location: umn/source + service_type: cbr + title: User Guide + type: umn + - hc_location: api2/cce + html_location: docs/cce/api-ref + link: /cloud-container-engine/api-ref/ + pdf_name: cce-api-ref + rst_location: api-ref/source + service_type: cce + title: API Reference + type: api-ref + - hc_location: usermanual2/cce + html_location: docs/cce/umn + link: /cloud-container-engine/umn/ + rst_location: umn/source + service_type: cce + title: User Guide + type: umn + - hc_location: api/ces + html_location: docs/ces/api-ref + link: /cloud-eye/api-ref/ + pdf_name: ces-api-ref + rst_location: api-ref/source + service_type: ces + title: API Reference + type: api-ref + - hc_location: usermanual/ces + html_location: docs/ces/umn + link: /cloud-eye/umn/ + pdf_name: ces-umn + rst_location: umn/source + service_type: ces + title: User Guide + type: umn + - hc_location: api/cts + html_location: docs/cts/api-ref + link: /cloud-trace-service/api-ref/ + pdf_name: cts-api-ref + rst_location: api-ref/source + service_type: cts + title: API Reference + type: api-ref + - hc_location: usermanual/cts + html_location: docs/cts/umn + link: /cloud-trace-service/umn/ + pdf_name: cts-umn + rst_location: umn/source + service_type: cts + title: User Guide + type: umn + - hc_location: api/dc + html_location: docs/dc/api-ref + link: /direct-connect/api-ref/ + pdf_name: dc-api-ref + rst_location: api-ref/source + service_type: dc + title: API Reference + type: api-ref + - hc_location: usermanual/dc + html_location: docs/dc/umn + link: /direct-connect/umn/ + pdf_name: dc-umn + rst_location: umn/source + service_type: dc + title: User Guide + type: umn + - hc_location: api/deh + html_location: docs/deh/api-ref + link: /dedicated-host/api-ref/ + pdf_name: deh-api-ref + rst_location: api-ref/source + service_type: deh + title: API Reference + type: api-ref + - hc_location: usermanual/deh + html_location: docs/deh/umn + link: /dedicated-host/umn/ + pdf_name: deh-umn + rst_location: umn/source + service_type: deh + title: User Guide + type: umn + - hc_location: api/dns + html_location: docs/dns/api-ref + link: /domain-name-service/api-ref/ + pdf_name: dns-api-ref + rst_location: api-ref/source + service_type: dns + title: API Reference + type: api-ref + - hc_location: usermanual/dns + html_location: docs/dns/umn + link: /domain-name-service/umn/ + pdf_name: dns-umn + rst_location: umn/source + service_type: dns + title: User Guide + type: umn + - hc_location: api/ecs + html_location: docs/ecs/api-ref + link: /elastic-cloud-server/api-ref/ + pdf_name: ecs-api-ref + rst_location: api-ref/source + service_type: ecs + title: API Reference + type: api-ref + - hc_location: devg/ecs + html_location: docs/ecs/dev + link: /elastic-cloud-server/dev-guide/ + pdf_name: ecs-dev-guide + rst_location: dev_guide/source + service_type: ecs + title: Developer Guide + type: dev + - hc_location: usermanual/ecs + html_location: docs/ecs/umn + link: /elastic-cloud-server/umn/ + pdf_name: ecs-umn + rst_location: umn/source + service_type: ecs + title: User Guide + type: umn + - hc_location: api/eip + html_location: docs/eip/api-ref + link: /elastic-ip/api-ref/ + pdf_name: eip-api-ref + rst_location: api-ref/source + service_type: eip + title: API Reference + type: api-ref + - hc_location: usermanual/eip + html_location: docs/eip/umn + link: /elastic-ip/umn/ + pdf_name: eip-umn + rst_location: umn/source + service_type: eip + title: User Guide + type: umn + - hc_location: api/elb + html_location: docs/elb/api-ref + link: /elastic-load-balancing/api-ref/ + pdf_name: elb-api-ref + rst_location: api-ref/source + service_type: elb + title: API Reference + type: api-ref + - hc_location: usermanual/elb + html_location: docs/elb/umn + link: /elastic-load-balancing/umn/ + pdf_name: elb-umn + rst_location: umn/source + service_type: elb + title: User Guide + type: umn + - hc_location: api/evs + html_location: docs/evs/api-ref + link: /elastic-volume-service/api-ref/ + pdf_name: evs-api-ref + rst_location: api-ref/source + service_type: evs + title: API Reference + type: api-ref + - hc_location: devg/evs + html_location: docs/evs/dev + link: /elastic-volume-service/dev-guide/ + pdf_name: evs-dev-guide + rst_location: dev_guide/source + service_type: evs + title: Developer Guide + type: dev + - hc_location: usermanual/evs + html_location: docs/evs/umn + link: /elastic-volume-service/umn/ + pdf_name: evs-umn + rst_location: umn/source + service_type: evs + title: User Guide + type: umn + - hc_location: api/iam + html_location: docs/iam/api-ref + link: /identity-access-management/api-ref/ + pdf_name: iam-api-ref + rst_location: api-ref/source + service_type: iam + title: API Reference + type: api-ref + - hc_location: usermanual/iam + html_location: docs/iam/umn + link: /identity-access-management/umn/ + pdf_name: iam-umn + rst_location: umn/source + service_type: iam + title: User Guide + type: umn + - hc_location: api/ims + html_location: docs/ims/api-ref + link: /image-management-service/api-ref/ + pdf_name: ims-api-ref + rst_location: api-ref/source + service_type: ims + title: API Reference + type: api-ref + - hc_location: devg/ims + html_location: docs/ims/dev + link: /image-management-service/dev-guide/ + pdf_name: ims-dev-guide + rst_location: dev_guide/source + service_type: ims + title: Developer Guide + type: dev + - html_location: docs/ims/public-images + link: /image-management-service/public-images/ + pdf_name: ims-public-images + rst_location: doc/public-images/source + service_type: ims + title: Public Image Introduction + type: public-images + - hc_location: usermanual/ims + html_location: docs/ims/umn + link: /image-management-service/umn/ + pdf_name: ims-umn + rst_location: umn/source + service_type: ims + title: User Guide + type: umn + - hc_location: api/kms + html_location: docs/kms/api-ref + link: /key-management-service/api-ref/ + pdf_name: kms-api-ref + rst_location: api-ref/source + service_type: kms + title: API Reference + type: api-ref + - hc_location: usermanual/kms + html_location: docs/kms/umn + link: /key-management-service/umn/ + pdf_name: kms-umn + rst_location: umn/source + service_type: kms + title: User Guide + type: umn + - hc_location: api/nat + html_location: docs/natgw/api-ref + link: /nat-gateway/api-ref/ + pdf_name: natgw-api-ref + rst_location: api-ref/source + service_type: natgw + title: API Reference + type: api-ref + - hc_location: usermanual/nat + html_location: docs/natgw/umn + link: /nat-gateway/umn/ + pdf_name: natgw-umn + rst_location: umn/source + service_type: natgw + title: User Guide + type: umn + - hc_location: api/obs + html_location: docs/obs/api-ref + link: /object-storage-service/api-ref/ + pdf_name: obs-api-ref + rst_location: api-ref/source + service_type: obs + title: API Reference + type: api-ref + - hc_location: browsertg/obs + html_location: docs/obs/tool + link: /object-storage-service/tool-guide/ + pdf_name: obs-tool-guide + rst_location: doc/tool/source + service_type: obs + title: Tool Guide (OBS Browser) + type: tool-guide + - hc_location: pfs/obs + html_location: docs/obs/pfs + link: /object-storage-service/parallel-file-system/ + pdf_name: obs-pfs + rst_location: doc/parallel-file-system/source + service_type: obs + title: Parallel File System (PFS) + type: parallel-file-system + - html_location: docs/obs/perms-cfg + link: /object-storage-service/permissions-configuration-guide/ + pdf_name: obs-perms-cfg + rst_location: doc/permissions-configuration-guide/source + service_type: obs + title: Permissions Configuration Guide + type: permissions-configuration-guide + - hc_location: api_swift/obs + html_location: docs/obs/api-swift + link: /object-storage-service/swiftapi/ + pdf_name: obs-swiftapi + rst_location: doc/swiftapi/source + service_type: obs + title: API Reference (Swift) + type: swiftapi + - hc_location: api_obs/obs + html_location: docs/obs/s3api + link: /object-storage-service/s3api/ + pdf_name: obs-s3api + rst_location: doc/s3api/source + service_type: obs + title: S3 API Reference + type: s3api + - hc_location: usermanual/obs + html_location: docs/obs/umn + link: /object-storage-service/umn/ + pdf_name: obs-umn + rst_location: umn/source + service_type: obs + title: User Guide + type: umn + - hc_location: api/rds + html_location: docs/rds/api-ref + link: /relational-database-service/api-ref/ + pdf_name: rds-api-ref + rst_location: api-ref/source + service_type: rds + title: API Reference + type: api-ref + - hc_location: usermanual/rds + html_location: docs/rds/umn + link: /relational-database-service/umn/ + pdf_name: rds-umn + rst_location: umn/source + service_type: rds + title: User Guide + type: umn + - hc_location: usermanual/sd + html_location: docs/sd/umn + link: /status-dashboard/umn/ + pdf_name: sd-umn + rst_location: umn/source + service_type: sd + title: User Guide + type: umn + - hc_location: api/sdrs + html_location: docs/sdrs/api-ref + link: /storage-disaster-recovery-service/api-ref/ + pdf_name: sdrs-api-ref + rst_location: api-ref/source + service_type: sdrs + title: API Reference + type: api-ref + - hc_location: usermanual/sdrs + html_location: docs/sdrs/umn + link: /storage-disaster-recovery-service/umn/ + pdf_name: sdrs-umn + rst_location: umn/source + service_type: sdrs + title: User Guide + type: umn + - hc_location: api/smn + html_location: docs/smn/api-ref + link: /simple-message-notification/api-ref/ + pdf_name: smn-api-ref + rst_location: api-ref/source + service_type: smn + title: API Reference + type: api-ref + - hc_location: usermanual/smn + html_location: docs/smn/umn + link: /simple-message-notification/umn/ + pdf_name: smn-umn + rst_location: umn/source + service_type: smn + title: User Guide + type: umn + - hc_location: api/swr + html_location: docs/swr/api-ref + link: /software-repository-container/api-ref/ + pdf_name: swr-api-ref + rst_location: api-ref/source + service_type: swr + title: API Reference + type: api-ref + - hc_location: usermanual/swr + html_location: docs/swr/umn + link: /software-repository-container/umn/ + pdf_name: swr-umn + rst_location: umn/source + service_type: swr + title: User Guide + type: umn + - hc_location: api/tms + html_location: docs/tms/api-ref + link: /tag-management-service/api-ref/ + pdf_name: tms-api-ref + rst_location: api-ref/source + service_type: tms + title: API Reference + type: api-ref + - hc_location: usermanual/tms + html_location: docs/tms/umn + link: /tag-management-service/umn/ + pdf_name: tms-umn + rst_location: umn/source + service_type: tms + title: User Guide + type: umn + - hc_location: api/vpc + html_location: docs/vpc/api-ref + link: /virtual-private-cloud/api-ref/ + pdf_name: vpc-api-ref + rst_location: api-ref/source + service_type: vpc + title: API Reference + type: api-ref + - hc_location: usermanual/vpc + html_location: docs/vpc/umn + link: /virtual-private-cloud/umn/ + pdf_name: vpc-umn + rst_location: umn/source + service_type: vpc + title: User Guide + type: umn + - hc_location: api/vpcep + html_location: docs/vpcep/api-ref + link: /vpc-endpoint/api-ref/ + pdf_name: vpcep-api-ref + rst_location: api-ref/source + service_type: vpcep + title: API Reference + type: api-ref + - hc_location: usermanual/vpcep + html_location: docs/vpcep/umn + link: /vpc-endpoint/umn/ + pdf_name: vpcep-umn + rst_location: umn/source + service_type: vpcep + title: User Guide + type: umn + - hc_location: api/vpn + html_location: docs/vpn/api-ref + link: /virtual-private-network/api-ref/ + pdf_name: vpn-api-ref + rst_location: api-ref/source + service_type: vpn + title: API Reference + type: api-ref + - hc_location: usermanual/vpn + html_location: docs/vpn/umn + link: /virtual-private-network/umn/ + pdf_name: vpn-umn + rst_location: umn/source + service_type: vpn + title: User Guide + type: umn + - environment: internal + hc_location: api/dwaf + html_location: docs/wafd/api-ref + link: /web-application-firewall-dedicated/api-ref/ + pdf_name: wafd-api-ref + rst_location: api-ref/source + service_type: wafd + title: API Reference + type: api-ref + - environment: internal + hc_location: usermanual/dwaf + html_location: docs/wafd/umn + link: /web-application-firewall-dedicated/umn/ + pdf_name: wafd-umn + rst_location: umn/source + service_type: wafd + title: User Guide + type: umn +service_categories: + - name: application + title: Application Services + - name: big_data + title: Big Data and Data Analysis + - name: compute + title: Compute + - name: container + title: Container + - name: database + title: Database + - name: network + title: Networking + - name: md + title: Management & Deployment + - name: security-services + title: Security Services + - name: storage + title: Storage + - name: other + title: Other +services: + - repositories: + - environment: internal + repo: docs/application-operations-management + type: gitea + - environment: public + repo: opentelekomcloud-docs/application-operations-management + type: github + service_category: application + service_title: Application Operations Management + service_type: aom + service_uri: application-operations-management + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/auto-scaling + type: gitea + - environment: public + repo: opentelekomcloud-docs/auto-scaling + type: github + service_category: compute + service_title: Auto Scaling + service_type: as + service_uri: auto-scaling + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-backup-recovery + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-backup-recovery + type: github + service_category: storage + service_title: Cloud Backup and Recovery + service_type: cbr + service_uri: cloud-backup-recovery + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-container-engine + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-container-engine + type: github + service_category: container + service_title: Cloud Container Engine + service_type: cce + service_uri: cloud-container-engine + environment: public + teams: + - name: docs-container-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-eye + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-eye + type: github + service_category: md + service_title: Cloud Eye + service_type: ces + service_uri: cloud-eye + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-trace-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-trace-service + type: github + service_category: md + service_title: Cloud Trace Service + service_type: cts + service_uri: cloud-trace-service + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/direct-connect + type: gitea + - environment: public + repo: opentelekomcloud-docs/direct-connect + type: github + service_category: network + service_title: Direct Connect + service_type: dc + service_uri: direct-connect + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/dedicated-host + type: gitea + - environment: public + repo: opentelekomcloud-docs/dedicated-host + type: github + service_category: compute + service_title: Dedicated Host + service_type: deh + service_uri: dedicated-host + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/domain-name-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/domain-name-service + type: github + service_category: network + service_title: Domain Name Service + service_type: dns + service_uri: domain-name-service + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - repo: docs/elastic-cloud-server + type: gitea + environment: internal + teams: + - name: docs-compute-ro + permission: read + - name: docs-compute-rw + permission: write + assignees: + - docs-compute-ro + - repo: opentelekomcloud-docs/elastic-cloud-server + type: github + environment: public + service_category: compute + service_title: Elastic Cloud Server + service_type: ecs + service_uri: elastic-cloud-server + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/elastic-ip + type: gitea + - environment: public + repo: opentelekomcloud-docs/elastic-ip + type: github + service_category: network + service_title: Elastic IP + service_type: eip + service_uri: elastic-ip + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/elastic-load-balancing + type: gitea + - environment: public + repo: opentelekomcloud-docs/elastic-load-balancing + type: github + service_category: network + service_title: Elastic Load Balancing + service_type: elb + service_uri: elastic-load-balancing + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/elastic-volume-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/elastic-volume-service + type: github + service_category: storage + service_title: Elastic Volume Service + service_type: evs + service_uri: elastic-volume-service + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/identity-access-management + type: gitea + - environment: public + repo: opentelekomcloud-docs/identity-access-management + type: github + service_category: security-services + service_title: Identity and Access Management + service_type: iam + service_uri: identity-access-management + environment: public + teams: + - name: docs-security-services-rw + permission: write + - repositories: + - environment: internal + repo: docs/image-management-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/image-management-service + type: github + service_category: compute + service_title: Image Management Service + service_type: ims + service_uri: image-management-service + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/key-management-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/key-management-service + type: github + service_category: security-services + service_title: Key Management Service + service_type: kms + service_uri: key-management-service + environment: public + teams: + - name: docs-security-services-rw + permission: write + - repositories: + - environment: internal + repo: docs/nat-gateway + type: gitea + - environment: public + repo: opentelekomcloud-docs/nat-gateway + type: github + service_category: network + service_title: NAT Gateway + service_type: natgw + service_uri: nat-gateway + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/object-storage-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/object-storage-service + type: github + service_category: storage + service_title: Object Storage Service + service_type: obs + service_uri: object-storage-service + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/relational-database-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/relational-database-service + type: github + service_category: database + service_title: Relational Database Service + service_type: rds + service_uri: relational-database-service + environment: public + teams: + - name: docs-database-rw + permission: write + - repositories: + - environment: internal + repo: docs/status-dashboard + type: gitea + - environment: public + repo: opentelekomcloud-docs/status-dashboard + type: github + service_category: other + service_title: Status Dashboard + service_type: sd + service_uri: status-dashboard/umn + environment: hidden + teams: + - name: docs-eco-rw + permission: write + - repositories: + - environment: internal + repo: docs/storage-disaster-recovery-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/storage-disaster-recovery-service + type: github + service_category: storage + service_title: Storage Disaster Recovery Service + service_type: sdrs + service_uri: storage-disaster-recovery-service + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/simple-message-notification + type: gitea + - environment: public + repo: opentelekomcloud-docs/simple-message-notification + type: github + service_category: application + service_title: Simple Message Notification + service_type: smn + service_uri: simple-message-notification + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/software-repository-container + type: gitea + - environment: public + repo: opentelekomcloud-docs/software-repository-container + type: github + service_category: container + service_title: Software Repository for Containers + service_type: swr + service_uri: software-repository-container + environment: public + teams: + - name: docs-container-rw + permission: write + - repositories: + - environment: internal + repo: docs/tag-management-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/tag-management-service + type: github + service_category: md + service_title: Tag Management Service + service_type: tms + service_uri: tag-management-service + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/virtual-private-cloud + type: gitea + - environment: public + repo: opentelekomcloud-docs/virtual-private-cloud + type: github + service_category: network + service_title: Virtual Private Cloud + service_type: vpc + service_uri: virtual-private-cloud + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/vpc-endpoint + type: gitea + - environment: public + repo: opentelekomcloud-docs/vpc-endpoint + type: github + service_category: network + service_title: VPC Endpoint + service_type: vpcep + service_uri: vpc-endpoint + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/virtual-private-network + type: gitea + - environment: public + repo: opentelekomcloud-docs/virtual-private-network + type: github + service_category: network + service_title: Virtual Private Network + service_type: vpn + service_uri: virtual-private-network + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/web-application-firewall-dedicated + type: gitea + - environment: public + repo: opentelekomcloud-docs/web-application-firewall-dedicated + type: github + service_category: security-services + service_title: Dedicated Web Application Firewall + service_type: wafd + service_uri: web-application-firewall-dedicated + environment: public + teams: + - name: docs-security-services-rw + permission: write diff --git a/otc_metadata/docs.py b/otc_metadata/docs.py new file mode 100644 index 0000000..733add1 --- /dev/null +++ b/otc_metadata/docs.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +import otc_metadata.data + +__all__ = ['Service'] + +BUILTIN_DATA = otc_metadata.data.read_data('docs.yaml') + + +def _normalize_type(service_type): + if service_type: + return service_type.replace('_', '-') + + +class Service(object): + """Encapsulation of the OTC Docs data + """ + + def __init__(self): + self._service_data = BUILTIN_DATA + + @property + def all_services(self): + "Service Categories data listing." + return copy.deepcopy(self._service_data['services']) diff --git a/otc_metadata/services.py b/otc_metadata/services.py new file mode 100644 index 0000000..195b067 --- /dev/null +++ b/otc_metadata/services.py @@ -0,0 +1,230 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import warnings + +import otc_metadata.data + +__all__ = ["Services"] + +BUILTIN_DATA = otc_metadata.data.read_data("services.yaml") + + +def _normalize_type(service_type): + if service_type: + return service_type.replace("_", "-") + + +class Services(object): + """Encapsulation of the OTC Services data""" + + def __init__(self): + self._service_data = BUILTIN_DATA + + def _sort_data(self): + # Sort every doc item by key + sorted_docs = [] + for doc in self._service_data["documents"]: + sorted_docs.append(dict(sorted(doc.items(), key=lambda kv: kv[0]))) + + # sort docs list by _ + self._service_data["documents"] = sorted( + sorted_docs, + key=lambda x: f"{x.get('service_type')}{x.get('title')}" + ) + # sort services by <service_type>_<service_title> + self._service_data["services"] = sorted( + self._service_data["services"], + key=lambda x: f"{x.get('service_type')}{x.get('service_title')}", + ) + + def _rewrite_data(self): + otc_metadata.data.rewrite_data("services.yaml", self._service_data) + + @property + def all_services(self): + "Service Categories data listing." + return copy.deepcopy(self._service_data["services"]) + + @property + def all_docs(self): + "Service Docs data listing." + return copy.deepcopy(self._service_data["documents"]) + + @property + def service_dict(self): + "Service Docs data listing." + res = dict() + for srv in self.all_services: + res[srv["service_type"]] = copy.deepcopy(srv) + return res + + @property + def service_categories(self): + """List services categories""" + res = [] + for cat in self._service_data["service_categories"]: + res.append(copy.deepcopy(cat)) + return res + + def services_by_category(self, category): + """List services matching category""" + res = [] + for srv in self.all_services: + if srv["service_category"] == category: + res.append(copy.deepcopy(srv)) + return res + + def services_with_docs_by_category(self, category, environment=None): + """Retrieve service category docs data + + :param str category: Optional Category filter + :param str env: Optional service environment. Influeces "repository" + field + """ + res = dict() + services = self.service_dict + for doc in self.all_docs: + cat = doc["service_type"] + service = services.get(cat) + if not service: + warnings.warn("No Service defition of type %s" % (cat)) + continue + if category and service["service_category"] != category: + continue + res.setdefault(cat, service) + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + if "environment" in doc and doc["environment"] != environment: + continue + res[cat].setdefault("docs", []) + res[cat]["docs"].append(res_doc) + return res + + def docs_by_service_category(self, category, environment=None): + """List services matching category + + :param str category: Category name + :param str env: Optional service environment. Influeces "repository" + field + """ + res = [] + services = self.service_dict + for doc in self.all_docs: + cat = doc["service_type"] + service = services.get(cat) + if not service: + warnings.warn("No Service defition of type %s" % (cat)) + continue + if service["service_category"] == category: + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + for srv_env in service["repositories"]: + if srv_env.get("environment") == environment: + res_doc["repository"] = srv_env["repo"] + res.append(res_doc) + return res + + def docs_by_service_type(self, service_type): + """List documents of the service + + :param str service_type: Service type + :returns: generator for documents + """ + for doc in self.all_docs: + if doc["service_type"] != service_type: + continue + yield copy.deepcopy(doc) + + def all_docs_full(self, environment): + """Return list or documents with full service data""" + services = self.service_dict + for doc in self.all_docs: + if not doc["service_type"] in services: + print(f"No service type {doc['service_type']}") + continue + service = services[doc["service_type"]] + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + for srv_env in service["repositories"]: + if srv_env.get("environment") == environment: + res_doc["repository"] = srv_env["repo"] + for srv_assignees in service.get("assignees", []): + if srv_assignees.get("environment") == environment: + res_doc["assignees"] = srv_assignees["names"] + yield res_doc + + def docs_html_by_category(self, environment): + """Generate structure for doc-exports repository""" + doc_struct = dict() + for srv in self.all_services: + doc_struct.setdefault(srv["service_category"], []) + srv_res = dict( + service_title=srv["service_title"], + service_type=srv["service_type"], + docs=[], + ) + if "teams" in srv: + srv_res["teams"] = [ + x for x in srv["teams"] if x["permission"] == "write" + ] + if "repositories" in srv and environment: + for repo in srv["repositories"]: + if ( + "environment" in repo + and repo["environment"] == environment + ): + srv_res["repository"] = repo["repo"] + for doc in self.all_docs: + if ( + "html_location" in doc + and doc["service_type"] == srv_res["service_type"] + ): + doc_res = dict( + html_location=doc["html_location"], + rst_location=doc["rst_location"], + title=doc["title"], + type=doc.get("type", "dummy"), + ) + if "pdf_name" in doc: + doc_res["pdf_name"] = doc["pdf_name"] + if "hc_location" in doc: + doc_res["hc_location"] = doc["hc_location"] + srv_res["docs"].append(doc_res) + if len(srv_res["docs"]) > 0: + doc_struct[srv["service_category"]].append(srv_res) + + return dict(categories=doc_struct) + + def get_service_with_docs_by_service_type(self, service_type): + """Retrieve service and service docs by service_type + + :param str service_type: Filter by service_type + """ + res = dict() + res['service'] = {} + docs = [] + services = self._service_data + for doc in services['documents']: + if doc['service_type'] == service_type: + docs.append(doc) + res['documents'] = docs + for service in services['services']: + if service['service_type'] == service_type: + res['service'] = service + break + return res diff --git a/otc_metadata/templates/conf.py.j2 b/otc_metadata/templates/conf.py.j2 new file mode 100644 index 0000000..3228eec --- /dev/null +++ b/otc_metadata/templates/conf.py.j2 @@ -0,0 +1,122 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# !!! +# This file is generated out of template in doc-exports repository. +# Beware overwriting it locally. + +import os +import sys + +extensions = [ + 'otcdocstheme', +{%- if otc_sbv %} + 'otc_sphinx_directives' +{%- endif %} +] + +otcdocs_auto_name = False +otcdocs_auto_version = False + +project = '{{ project }}' +otcdocs_repo_name = '{{ repo_name }}' +# Those variables are required for edit/bug links +{%- if git_fqdn %} +otcdocs_git_fqdn = '{{ git_fqdn }}' +{%- endif %} +{%- if git_type %} +otcdocs_git_type = '{{ git_type }}' +{%- endif %} + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) + +# -- General configuration ---------------------------------------------------- +# https://docutils.sourceforge.io/docs/user/smartquotes.html - it does not +# what it is expected +smartquotes = False + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +copyright = u'2022-present, Open Telekom Cloud' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +language = 'en' + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { +{%- if html_options is defined -%} +{%- for (k, v) in html_options.items() %} +{%- if v is boolean %} + "{{ k }}": {{ v }}, +{%- else %} + "{{ k }}": "{{ v }}", +{%- endif %} +{%- endfor %} +{%- endif %} +} + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +{% if title is defined %} +html_title = "{{ title }}" +{% endif %} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Do not include sources into the rendered results +html_copy_source = False + +# -- Options for PDF output -------------------------------------------------- +latex_documents = [ +{%- if pdf_name is defined %} + ('index', + '{{ pdf_name }}.tex', + u'{{ title }}', + u'OpenTelekomCloud', 'manual'), +{% endif -%} +] + diff --git a/otc_metadata/templates/doc_requirements.txt.j2 b/otc_metadata/templates/doc_requirements.txt.j2 new file mode 100644 index 0000000..87daf92 --- /dev/null +++ b/otc_metadata/templates/doc_requirements.txt.j2 @@ -0,0 +1,7 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +otcdocstheme # Apache-2.0 +# releasenotes +reno>=3.1.0 # Apache-2.0 + +otc-sphinx-directives>=0.1.0 +git+https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-swiss.git#egg=otc_metadata diff --git a/otc_metadata/templates/index_sbv.rst.j2 b/otc_metadata/templates/index_sbv.rst.j2 new file mode 100644 index 0000000..1ef5b68 --- /dev/null +++ b/otc_metadata/templates/index_sbv.rst.j2 @@ -0,0 +1,7 @@ +{{ sbv_title }} + +.. directive_wrapper:: + :class: container-sbv + + .. service_card:: + :service_type: {{ service_type }} diff --git a/otc_metadata/templates/tox.ini.j2 b/otc_metadata/templates/tox.ini.j2 new file mode 100644 index 0000000..e6ecf9d --- /dev/null +++ b/otc_metadata/templates/tox.ini.j2 @@ -0,0 +1,125 @@ +[tox] +minversion = 3.1 +envlist = py39,pep8 +skipsdist = True +ignore_basepython_conflict = True + +[testenv] +usedevelop = True +install_command = pip install {opts} {packages} +deps = + -r{toxinidir}/requirements.txt +commands = stestr run {posargs} + stestr slowest + +[testenv:pep8] +commands = + doc8 doc/source README.rst + +[testenv:venv] +deps = + -r{toxinidir}/requirements.txt +commands = {posargs} + +# This env is invoked in the periodic pipeline and is therefore responsible to +# build all relevant docs at once. +[testenv:docs] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + mkdir + cp + sh + rm + sphinx-build +commands = + rm -rf doc/build/html doc/build/doctrees + sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html +{%- for doc in docs %} + {[testenv:{{ doc.type }}]commands} + {[testenv:json-{{ doc.type }}]commands} +{%- endfor %} + +[testenv:docs-pdf] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + rm + mkdir + make + bash + cp +commands = + mkdir -p doc/build/pdf +{%- for doc in docs %} +{%- if doc.pdf_name %} + {[testenv:{{ doc.type }}-pdf-docs]commands} +{%- endif %} +{%- endfor %} + +{% for doc in docs -%} +{% set loc = doc.rst_location | replace('/source', '') %} +# HTML version +[testenv:{{ doc.type }}] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + cp + mkdir +commands = + sphinx-build -W --keep-going -b html {{ loc }}/source doc/build/html/{{ doc.type }} +{%- if doc.type == 'api-ref' %} + mkdir -p api-ref/build/html + cp -av doc/build/html/api-ref api-ref/build/html +{%- elif doc.type == 'umn' %} + mkdir -p umn/build/html + cp -av doc/build/html/umn umn/build/html +{%- elif doc.type == 'dev-guide' %} + mkdir -p dev_guide/build/html + cp -av doc/build/html/dev-guide dev_guide/build/html +{%- endif %} + +# Json version (for search) +[testenv:json-{{ doc.type }}] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + cp + mkdir + sh +commands = + sphinx-build -W --keep-going -b json {{ loc }}/source doc/build/json/{{ doc.type }} + # Drop data useless for the search - wrap it also with sh/xargs due to bugs + # in tox + sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf" +{%- if doc.type == 'api-ref' %} + mkdir -p api-ref/build/json + cp -av doc/build/json/api-ref api-ref/build/json +{%- elif doc.type == 'umn' %} + mkdir -p umn/build/json + cp -av doc/build/json/umn umn/build/json +{%- elif doc.type == 'dev-guide' %} + mkdir -p dev_guide/build/json + cp -av doc/build/json/dev-guide dev_guide/build/json +{%- endif %} + +{% if doc.pdf_name -%} +# PDF version +[testenv:{{ doc.type }}-pdf-docs] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + rm + mkdir + make + bash + cp +commands = + rm -rf {{ loc }}/build/pdf + sphinx-build -W --keep-going -b latex {{ loc }}/source {{ loc }}/build/pdf/ + bash -c "for f in {{ loc }}/build/pdf/*.gif; do convert $f $\{f/%gif/png\}; done || true" + bash -c "for f in {{ loc }}/build/pdf/*.tex; do sed -iorig 's/\.gif//g' $f; done" + make -C {{ loc }}/build/pdf + mkdir -p doc/build/pdf + cp {{ loc }}/build/pdf/{{ doc.pdf_name }}.pdf doc/build/pdf/ +{% endif %} +{% endfor %} + +[doc8] +ignore = D001 +extensions = .rst, .yaml \ No newline at end of file diff --git a/otc_metadata/tests/__init__.py b/otc_metadata/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/otc_metadata/tests/test_otc_metadata.py b/otc_metadata/tests/test_otc_metadata.py new file mode 100644 index 0000000..609199f --- /dev/null +++ b/otc_metadata/tests/test_otc_metadata.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +test_otc-metadata +---------------------------------- + +Tests for `otc-metadata` module. +""" + +from unittest import TestCase + +from otc_metadata import services + + +class TestOtcMetadata(TestCase): + def setUp(self): + self.data = services.Services() + + def test_data_is_sorted(self): + curr = self.data + new = services.Services() + new._sort_data() + self.assertEqual( + curr._service_data, new._service_data, "Data is sorted properly" + ) + + def test_service_categories(self): + category = dict() + for cat in self.data._service_data["service_categories"]: + category[cat["name"]] = cat["title"] + for srv in self.data.all_services: + self.assertTrue( + srv["service_category"] in category, + f"Category {srv['service_category']} is present", + ) + + def test_doc_contains_required_data(self): + srv_types = dict() + for srv in self.data.all_services: + srv_types[srv["service_type"]] = srv + for doc in self.data.all_docs: + for attr in [ + "rst_location", + "service_type", + "title", + "type", + ]: + self.assertIn(attr, doc, f"Document {doc} contains {attr}") + self.assertIn( + doc["service_type"], + srv_types, + f"Document {doc} contains valid service_type", + ) diff --git a/releasenotes/notes/.placeholder b/releasenotes/notes/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/_static/.placeholder b/releasenotes/source/_static/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/_templates/.placeholder b/releasenotes/source/_templates/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 0000000..ecd0d77 --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'otcdocstheme', + 'reno.sphinxext', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'otc-metadata Release Notes' +copyright = '2022, Open Telekom Cloud Developers' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +# The full version, including alpha/beta/rc tags. +release = '' +# The short X.Y version. +version = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'otc-metadataReleaseNotesdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'otc-metadataReleaseNotes.tex', + 'otc-metadata Release Notes Documentation', + 'Open Telekom Cloud', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'otc-metadatarereleasenotes', + 'otc-metadata Release Notes Documentation', + ['Open Telekom Cloud'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'otc-metadata ReleaseNotes', + 'otc-metadata Release Notes Documentation', + 'Open Telekom Cloud Foundation', 'otc-metadataReleaseNotes', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 0000000..a040fa0 --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,8 @@ +============================================ + otc-metadata Release Notes +============================================ + +.. toctree:: + :maxdepth: 1 + + unreleased diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst new file mode 100644 index 0000000..cd22aab --- /dev/null +++ b/releasenotes/source/unreleased.rst @@ -0,0 +1,5 @@ +============================== + Current Series Release Notes +============================== + +.. release-notes:: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1d18dd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +pbr>=2.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ad735b1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,25 @@ +[metadata] +name = otc-metadata +summary = Metadata about OTC for Ecosystem +description_file = + README.rst +author = Open Telekom Cloud +home_page = https://open.telekom.cloud/ +python_requires = >=3.6 +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: Implementation :: CPython + +[files] +packages = + otc_metadata diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7f1bbe8 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..8a2bd6e --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,7 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +stestr>=2.0.0 # Apache-2.0 +testtools>=2.2.0 # MIT +flake8 diff --git a/tools-requirements.txt b/tools-requirements.txt new file mode 100644 index 0000000..5fee53b --- /dev/null +++ b/tools-requirements.txt @@ -0,0 +1,6 @@ +GitPython +ruamel.yaml +requests +jinja2 +dirsync +cookiecutter diff --git a/tools/bootstrap_repositories.py b/tools/bootstrap_repositories.py new file mode 100644 index 0000000..719ad38 --- /dev/null +++ b/tools/bootstrap_repositories.py @@ -0,0 +1,156 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +# import os +import pathlib +import subprocess +# import warnings + +# from git import exc +from git import Repo +# from git import SymbolicReference + +# from jinja2 import PackageLoader +# from jinja2 import Environment +# from jinja2 import select_autoescape + +from cookiecutter.main import cookiecutter + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def create_repo(repo, repo_dir, service): + repo_owner, repo_name = repo["repo"].split("/") + git_repo = None + + if repo_dir.exists(): + logging.debug(f"Repository {repo['repo']} is already existing") + return + + logging.info(f"Creating repository {repo_owner}/{repo_name}") + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + git_fqdn = "github.com" + + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.debug("Error") + pass + + if git_repo: + return + + if repo["type"] == "gitea": + subprocess.run( + args=[ + "tea", + "repo", + "create", + "--name", + repo_name, + "--owner", + repo_owner, + ] + ) + elif repo["type"] == "github": + subprocess.run( + args=[ + "gh", + "repo", + "create", + repo["repo"], + "--public", + ] + ) + cookiecutter( + template="https://github.com/opentelekomcloud/docs-cookiecutter", + output_dir=repo_dir.parent, + no_input=True, + extra_context=dict( + git_fqdn=git_fqdn, + repo_group=repo_owner, + repo_name=repo_name, + project_short_description=( + f"Documentation project for {service['service_title']} " + "service" + ), + overwrite_if_exists=True, + ), + ) + git_repo = Repo(repo_dir) + git_repo.create_remote("origin", repo_url) + git_repo.remotes.origin.fetch() + git_repo.git.push("--set-upstream", "origin", "main") + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] != args.target_environment: + continue + + checkout_exists = repo_dir.exists() + logging.debug(f"Repository {repo} exists {checkout_exists}") + repo_dir.parent.mkdir(parents=True, exist_ok=True) + if True: # not checkout_exists: + create_repo(repo, repo_dir, service) + + +def main(): + parser = argparse.ArgumentParser(description="Bootstrap repositories.") + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/convert_data.py b/tools/convert_data.py new file mode 100644 index 0000000..1d7fb3e --- /dev/null +++ b/tools/convert_data.py @@ -0,0 +1,22 @@ +import re + +import otc_metadata.services +from ruamel.yaml import YAML + +data = otc_metadata.services.Services() +new_data = data._service_data + +# services = data.service_dict + +for doc in new_data["documents"]: + hc_location = None + link = doc.get("link") + if link: + print(f"Parsing {link}") + # (p1, p2) = link.split("/") + doc["link"] = re.sub(r"/(.*)/(.*)/", r"/\2/\1/", link) + +_yaml = YAML() +_yaml.indent(mapping=2, sequence=4, offset=2) +with open("new.yaml", "w") as fd: + _yaml.dump(new_data, fd) diff --git a/tools/generate_doc_confpy.py b/tools/generate_doc_confpy.py new file mode 100644 index 0000000..7c3a5c9 --- /dev/null +++ b/tools/generate_doc_confpy.py @@ -0,0 +1,363 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import pathlib +import requests +import subprocess + +from git import exc +from git import Repo + +from jinja2 import PackageLoader +from jinja2 import Environment +from jinja2 import select_autoescape + +import otc_metadata.services + +data = otc_metadata.services.Services() + +api_session = requests.Session() + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_to = None + repo_to = None + url_to = None + target_repo = None + git_fqdn = None + + env = Environment( + loader=PackageLoader("otc_metadata"), autoescape=select_autoescape() + ) + conf_py_template = env.get_template("conf.py.j2") + tox_ini_template = env.get_template("tox.ini.j2") + index_sbv_template = env.get_template("index_sbv.rst.j2") + doc_requirements_template = env.get_template("doc_requirements.txt.j2") + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] == args.target_environment: + copy_to = repo_dir + else: + logging.debug(f"Skipping repository {repo}") + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + repo_dir.rmdir() + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.error(f"Error cloning repository {repo_url}") + return + + if repo["environment"] == args.target_environment: + url_to = repo_url + repo_to = git_repo + target_repo = repo + break + + if not target_repo: + logging.info( + f"No repository service {service['service_title']}" + f"for environment {args.target_environment}" + ) + return + + branch_name = f"{args.branch_name}" + if args.branch_force: + logging.debug("Dropping current branch") + try: + repo_to.delete_head(branch_name, force=True) + repo_to.delete_head(f"refs/heads/{args.branch_name}", force=True) + except exc.GitCommandError as e: + print(e) + pass + try: + new_branch = repo_to.create_head(branch_name, "main") + except Exception as e: + logging.warning(f"Skipping service {service} due to {e}") + return + new_branch.checkout() + + service_docs = list(data.docs_by_service_type(service["service_type"])) + + for doc in service_docs: + logging.debug(f"Analyzing document {doc}") + + conf_py_path = pathlib.Path(copy_to, doc["rst_location"], "conf.py") + if not conf_py_path.exists(): + logging.info(f"Path for document {doc['title']} does not exist") + conf_py_path.parent.mkdir(parents=True, exist_ok=True) + context = dict( + repo_name=target_repo["repo"], + project=service["service_title"], + # pdf_name=doc["pdf_name"], + title=f"{service['service_title']} - {doc['title']}", + ) + if "pdf_name" in doc: + context["pdf_name"] = doc["pdf_name"] + if git_fqdn: + context["git_fqdn"] = git_fqdn + if target_repo.get("type") != "github": + context["git_type"] = target_repo["type"] + if args.target_environment == "internal": + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-int.otc-service.com", + ) + conf_py_content = conf_py_template.render(**context) + with open(conf_py_path, "w", encoding="utf-8", newline="") as out: + logging.debug(f"Generating {conf_py_path} from template...") + out.write(conf_py_content) + + repo_to.index.add([doc["rst_location"]]) + + if args.update_sbv: + """Add or update service-based-view""" + copy_path = pathlib.Path(copy_to, 'doc', 'source') + context = dict( + repo_name=target_repo["repo"], + project=service["service_title"], + # pdf_name=doc["pdf_name"], + title=f"{service['service_title']} - Service Based View", + service_type=service["service_type"] + ) + if not copy_path.exists(): + logging.info("Path for sbv does not exist") + copy_path.mkdir(parents=True, exist_ok=True) + context["otc_sbv"] = True + if git_fqdn: + context["git_fqdn"] = git_fqdn + if target_repo.get("type") != "github": + context["git_type"] = target_repo["type"] + if args.target_environment == "internal": + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-int.otc-service.com", + ) + sbv_title = (service["service_title"] + "\n" + + ('=' * len(service["service_title"]))) + context["sbv_title"] = sbv_title + conf_py_content = conf_py_template.render(**context) + index_sbv_content = index_sbv_template.render(**context) + with open( + pathlib.Path(copy_path, "conf.py"), + "w", + encoding="utf-8") as out: + out.write(conf_py_content) + repo_to.index.add(pathlib.Path(copy_path, "conf.py")) + + if (not args.overwrite_index_sbv + and pathlib.Path(copy_path, "index.rst").exists()): + logging.info("File index.rst for sbv exists. Skipping") + else: + with open( + pathlib.Path(copy_path, "index.rst"), + "w", + encoding="utf-8") as out: + out.write(index_sbv_content) + repo_to.index.add(pathlib.Path(copy_path, "index.rst")) + + placeholder_path = pathlib.Path(copy_path, "_static") + if not pathlib.Path(placeholder_path, "placeholder").exists(): + placeholder_path.mkdir(parents=True, exist_ok=True) + open(pathlib.Path(placeholder_path, "placeholder"), 'a').close() + repo_to.index.add(pathlib.Path(placeholder_path, "placeholder")) + + if args.update_tox: + """Update tox.ini""" + context = dict(docs=[]) + for doc in service_docs: + if doc["type"] == "dev": + doc["type"] = "dev-guide" + context["docs"].append(doc) + + tox_ini_content = tox_ini_template.render(**context) + tox_ini_path = pathlib.Path(copy_to, "tox.ini") + doc_requirements_content = doc_requirements_template.render(**context) + doc_requirements_path = pathlib.Path( + copy_to, "doc", "requirements.txt" + ) + doc_requirements_path.parent.mkdir(parents=True, exist_ok=True) + with open(tox_ini_path, "w", encoding="utf-8", newline="") as out: + logging.debug(f"Generating {tox_ini_path} from template...") + out.write(tox_ini_content) + repo_to.index.add(["tox.ini"]) + with open( + doc_requirements_path, "w", encoding="utf-8", newline="" + ) as out: + logging.debug( + f"Generating {doc_requirements_path} from template..." + ) + out.write(doc_requirements_content) + repo_to.index.add(["doc/requirements.txt"]) + + if len(repo_to.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % service["service_type"] + ) + return + repo_to.index.commit( + args.commit_description + ) + push_args = ["--set-upstream", "origin", branch_name] + if args.branch_force: + push_args.append("--force") + repo_to.git.push(*push_args) + if "github" in url_to: + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=False + ) + elif "gitea" in url_to and args.token: + open_pr( + args, + repo["repo"], + dict( + title="Update Docs configuration", + head=branch_name, + ), + ) + + +def open_pr(args, repository, pr_data): + req = dict( + base=pr_data.get("base", "main"), + head=pr_data["head"], + ) + if "title" in pr_data: + req["title"] = pr_data["title"] + if "body" in pr_data: + req["body"] = pr_data["body"].replace("\\n", "\n") + if "assignees" in pr_data: + req["assignees"] = pr_data["assignees"] + if "labels" in pr_data: + req["labels"] = pr_data["labels"] + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/pulls", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + # print(f"Going to open PR with title {pr_data['title']} in {repository}") + + +def main(): + parser = argparse.ArgumentParser( + description="Update conf.py file in repositories." + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--update-tox", action="store_true", help="Whether to update tox.ini." + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="confpy", + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--branch-force", + action="store_true", + help="Whether to force branch recreation.", + ) + parser.add_argument("--token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + parser.add_argument( + "--update-sbv", + action="store_true", + help="Whether to update service-based-view" + ) + parser.add_argument( + "--overwrite-index-sbv", + action="store_true", + help=("Whether to overwrite index.rst for service-based-view." + + "\nCan only be used if --update-sbv is also specified") + ) + parser.add_argument( + "--commit-description", + default=( + "Update tox.ini && conf.py file\n\n" + "Performed-by: gitea/infra/otc-metadata-swiss/" + "tools/generate_doc_confpy.py" + ), + help="Commit description for the commit", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.overwrite_index_sbv and not args.update_sbv: + logging.error( + "Cannot overwrite index.rst for service-based-view" + + " without updating service-based-view" + ) + exit(1) + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + if args.token: + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_doc_gitcontrol_repos.py b/tools/generate_doc_gitcontrol_repos.py new file mode 100644 index 0000000..bcb63a7 --- /dev/null +++ b/tools/generate_doc_gitcontrol_repos.py @@ -0,0 +1,184 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import copy +import logging +import yaml + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_services(args, services): + """Process services""" + gitea_bp = dict( + branch_name="main", + enable_approvals_whitelist=True, + approvals_whitelist_teams=["docs-infra-team"], + block_on_rejected_reviews=True, + dismiss_stale_approvals=True, + enable_push=False, + status_check_contexts=["gl/check"], + enable_merge_whitelist=True, + merge_whitelist_usernames=["zuul"], + ) + gitea_repo_template = dict( + default_branch="main", + description="Open Telekom Cloud Service docs", + homepage=None, + archived=False, + has_issues=True, + has_projects=False, + has_wiki=False, + default_delete_branch_after_merge=True, + allow_merge_commit=False, + allow_squash_merge=True, + allow_rebase_merge=False, + default_merge_style="squash", + branch_protections=[], + ) + github_bp = dict( + branch_name="main", + enable_approvals_whitelist=True, + approvals_whitelist_teams=["docs-infra-team"], + block_on_rejected_reviews=True, + dismiss_stale_approvals=True, + enable_push=False, + status_check_contexts=["gl/check"], + enable_merge_whitelist=True, + merge_whitelist_usernames=["zuul"], + ) + github_repo_template = dict( + default_branch="main", + description="Open Telekom Cloud Service docs", + homepage=None, + archived=False, + has_issues=True, + has_projects=False, + has_wiki=False, + delete_branch_on_merge=True, + allow_merge_commit=False, + allow_squash_merge=True, + allow_rebase_merge=False, + allow_update_branch=True, + branch_protections=[], + ) + + for service in services: + logging.debug(f"Processing service {service}") + config = None + repo_name = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + if repo["environment"] != args.target_environment: + continue + + if repo["type"] == "gitea": + teams = [] + branch_protections_main = copy.deepcopy(gitea_bp) + if "teams" in repo: + teams_def = repo["teams"] + if "teams" in service: + teams_def = service["teams"] + if teams_def: + for team in teams_def: + branch_protections_main[ + "approvals_whitelist_teams" + ].append(team["name"]) + teams.append(team["name"]) + data = copy.deepcopy(gitea_repo_template) + data["description"] = ( + f"Open Telekom Cloud {service['service_title']} " + f"Service docs" + ) + data["branch_protections"].append(branch_protections_main) + data["teams"] = teams + repo_name = repo["repo"].split("/")[1] + config = {repo_name: data} + + elif repo["type"] == "github": + teams = [] + branch_protections_main = copy.deepcopy(github_bp) + if "teams" in repo: + teams_def = repo["teams"] + if "teams" in service: + teams_def = service["teams"] + if teams_def: + for team in teams_def: + teams.append( + { + "slug": team["name"], + "permission": "push" + if team["permission"] == "write" + else "pull", + } + ) + data = copy.deepcopy(github_repo_template) + data["description"] = ( + f"Open Telekom Cloud {service['service_title']} " f"docs" + ) + data["branch_protections"].append( + {"branch": "main", "template": "zuul"} + ) + data["teams"] = teams + repo_name = repo["repo"].split("/")[1] + config = {repo_name: data} + + else: + logging.error( + "Repository type %s is not supported", repo["type"] + ) + exit(1) + + if data: + with open( + f"{args.work_dir}/{repo_name}.yml", + "w", + encoding="utf-8", + newline="", + ) as out: + logging.debug( + "Generating %s config from template...", + service["service_type"], + ) + yaml.dump(config, out) + + +def main(): + parser = argparse.ArgumentParser( + description="Update conf.py file in repositories." + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + + process_services(args, data.all_services) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_docexports_data.py b/tools/generate_docexports_data.py new file mode 100644 index 0000000..8b8012a --- /dev/null +++ b/tools/generate_docexports_data.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys + +from ruamel.yaml import YAML + +import otc_metadata.services + + +def main(): + data = otc_metadata.services.Services() + + _yaml = YAML() + _yaml.indent(mapping=2, sequence=4, offset=2) + sys.stdout.write( + "# Auto-generated by otc_metadata.generate_docexports.data\n" + ) + _yaml.dump(data.docs_html_by_category("internal"), sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/tools/open_doc_issue.py b/tools/open_doc_issue.py new file mode 100644 index 0000000..bf4b34b --- /dev/null +++ b/tools/open_doc_issue.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import argparse +# import re +import requests +# import sys + +import otc_metadata.services + +api_session = requests.Session() + + +def open_issue(args, repository, issue_data): + req = dict( + title=issue_data["title"], body=issue_data["body"].replace("\\n", "\n") + ) + if "assignees" in issue_data: + req["assignees"] = issue_data["assignees"] + if "labels" in issue_data: + req["labels"] = issue_data["labels"] + print(req) + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/issues", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + print( + f"Going to open issue with title {issue_data['title']} in {repository}" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Open Issue for every document." + ) + parser.add_argument("token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + parser.add_argument("--environment", help="Environment for the repository") + parser.add_argument("--title", required=True, help="Issue title") + parser.add_argument("--body", required=True, help="Issue body") + parser.add_argument( + "--repo", + help="Repository to report issue in (instead of doc repository).", + ) + parser.add_argument( + "--assignee", + help="Issue assignee to use instead of document service assignees.", + ) + parser.add_argument( + "--labels", + help="Issue labels to use (comma separated list of label IDs).", + ) + args = parser.parse_args() + data = otc_metadata.services.Services() + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for doc in data.all_docs_full(environment=args.environment): + issue_data = dict( + title=args.title.format(**doc), + body=args.body.format(**doc), + repository=doc["repository"], + ) + if "assignees" in doc: + issue_data["assignees"] = doc["assignees"] + if args.assignee: + issue_data["assignees"] = [args.assignee] + if args.labels: + issue_data["labels"] = [int(x) for x in args.labels.split(",")] + open_issue(args, args.repo or doc["repository"], issue_data) + + +if __name__ == "__main__": + main() diff --git a/tools/service_list b/tools/service_list new file mode 100644 index 0000000..b17a6f0 --- /dev/null +++ b/tools/service_list @@ -0,0 +1,50 @@ +antiddos +aom +as +bms +cbr +cce +cdn +ces +csbs +css +cts +das +dc +dcs +dds +deh +dis +dms +dns +drs +dws +ecs +eip +elb +evs +gaussdb_mysql +gaussdb_nosql +iam +ims +kms +lts +ma +mrs +natgw +obs +opengauss +plas +rds +rts +sdrs +sfs +smn +swr +tms +vbs +vpc +vpcep +vpn +waf +wafd diff --git a/tools/sort_metadata.py b/tools/sort_metadata.py new file mode 100644 index 0000000..caeae59 --- /dev/null +++ b/tools/sort_metadata.py @@ -0,0 +1,9 @@ +# import copy + +import otc_metadata.services +# from ruamel.yaml import YAML + +data = otc_metadata.services.Services() + +data._sort_data() +data._rewrite_data() diff --git a/tools/sync_doc_repo.py b/tools/sync_doc_repo.py new file mode 100644 index 0000000..5bdbc7b --- /dev/null +++ b/tools/sync_doc_repo.py @@ -0,0 +1,201 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +import os +import pathlib +import subprocess +import warnings +from dirsync import sync + +from git import Repo +from git import SymbolicReference + + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_repositories(args, service): + """Checkout repositories""" + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_from = None + copy_to = None + repo_to = None + url_from = None + url_to = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] == args.source_environment: + copy_from = repo_dir + elif repo["environment"] == args.target_environment: + copy_to = repo_dir + else: + continue + + checkout_exists = repo_dir.exists() + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + if not checkout_exists: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + else: + logging.debug("Checkout already exists") + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + + if repo["environment"] == args.source_environment: + url_from = repo_url + elif repo["environment"] == args.target_environment: + url_to = repo_url + repo_to = git_repo + logging.debug(f"Synchronizing from={url_from}, to={url_to}") + + if not copy_from or not copy_to: + logging.warn( + "Not synchronizing documents of the service. " + "Target or source not known." + ) + + for doc in data.docs_by_service_type(service["service_type"]): + logging.debug(f"Analyzing document {doc}") + if args.document_type and doc.get("type") != args.document_type: + logging.info( + f"Skipping synchronizing {doc['title']} " + f"due to the doc-type filter." + ) + continue + if doc.get("environment"): + logging.info( + f"Skipping synchronizing {doc['title']} " + f"since it is environment bound." + ) + continue + + branch_name = f"{args.branch_name}#{doc['type']}" + remote_ref = SymbolicReference.create( + repo_to, "refs/remotes/origin/%s" % branch_name + ) + new_branch = repo_to.create_head(branch_name, "main") + remote_ref = repo_to.remotes[0].refs[branch_name] # get correct type + new_branch.set_tracking_branch(remote_ref) + new_branch.checkout() + + source_path = pathlib.Path(copy_from, doc["rst_location"]) + target_path = pathlib.Path(copy_to, doc["rst_location"]) + sync( + source_path, + target_path, + 'sync', + purge=True, + create=True, + content=True, + ignore=['conf.py'] + ) + repo_to.index.add([doc["rst_location"]]) + + for obj in repo_to.index.diff(None).iter_change_type('D'): + repo_to.index.remove([obj.b_path]) + if len(repo_to.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug("No changes.") + continue + repo_to.index.commit( + ( + f"Synchronize {doc['title']}\n\n" + f"Overwriting document\n" + f"from: {url_from}\n" + f"to: {url_to}\n\n" + "Performed-by: gitea/infra/otc-metadata-swiss/tools/sync_doc_repo.py" + ) + ) + repo_to.remotes.origin.push(new_branch) + if "github" in url_to and args.open_pr_gh: + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Synchronize document between environments." + ) + parser.add_argument( + "--source-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--service-type", + required=True, + help="Service to which document(s) belongs to", + ) + parser.add_argument( + "--document-type", + help=( + "Type of the document to synchronize. " + "All will be synchronized if not set." + ), + ) + parser.add_argument( + "--branch-name", + required=True, + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--open-pr-gh", + action="store_true", + help="Open Pull Request using `gh` utility (need to be present).", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + service = data.service_dict.get(args.service_type) + if not service: + warnings.warn(f"Service {args.service_type} was not found in metadata") + os.exit(1) + + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/update_zuul_project_configs.py b/tools/update_zuul_project_configs.py new file mode 100644 index 0000000..8a898d9 --- /dev/null +++ b/tools/update_zuul_project_configs.py @@ -0,0 +1,294 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +# import os +import pathlib +import requests +import subprocess +# import warnings + +from git import exc +from git import Repo +# from git import SymbolicReference + +from ruamel.yaml import CommentedMap +from ruamel.yaml import YAML + +import otc_metadata.services + +data = otc_metadata.services.Services() + +yaml = YAML() + +api_session = requests.Session() + + +def load_zuul_config(workdir): + for path in [".zuul.yaml", "zuul.yaml"]: + test_path = pathlib.Path(workdir, path) + if test_path.exists(): + with open(test_path, "r") as f: + zuul_config = yaml.load(f) + return (zuul_config, test_path) + + +def open_pr(args, repository, pr_data): + req = dict( + title=pr_data["title"], + body=pr_data["body"].replace("\\n", "\n"), + base=pr_data.get("base", "main"), + head=pr_data["head"], + ) + if "assignees" in pr_data: + req["assignees"] = pr_data["assignees"] + if "labels" in pr_data: + req["labels"] = pr_data["labels"] + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/pulls", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + print(f"Going to open PR with title {pr_data['title']} in {repository}") + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_to = None + # repo_to = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] != args.environment: + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.update() + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + repo_dir.rmdir() + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.error(f"Error cloning repository {repo_url}") + return + + branch_name = f"{args.branch_name}" + if args.branch_force: + logging.debug("Dropping current branch") + try: + git_repo.delete_head(branch_name, force=True) + except exc.GitCommandError: + pass + try: + new_branch = git_repo.create_head(branch_name, "main") + except Exception as ex: + logging.warning(f"Skipping service {service} due to {ex}") + return + new_branch.checkout() + + (zuul_config, zuul_file_name) = load_zuul_config(repo_dir) + zuul_templates = None + zuul_jobs = dict() + zuul_new_jobs = list() + zuul_config_updated = False + for item in zuul_config: + if "project" in item.keys(): + project = item["project"] + zuul_templates = project.setdefault("templates", []) + if not zuul_templates: + zuul_templates = [] + elif "job" in item.keys(): + job = item["job"] + zuul_jobs[job["name"]] = job + print(f"Existing jobs {zuul_jobs}") + if "helpcenter-base-jobs" not in zuul_templates: + zuul_templates.append("helpcenter-base-jobs") + zuul_config_updated = True + + job_suffix = ( + "-hc-int-jobs" if args.environment == "internal" else "-hc-jobs" + ) + for doc in data.docs_by_service_type(service["service_type"]): + logging.debug(f"Analyzing document {doc}") + if not doc.get("type"): + continue + if doc["type"] == "dev": + doc_type = "dev-guide" + else: + doc_type = doc["type"] + template_name = f"{doc_type}{job_suffix}" + if doc_type in ["api-ref", "umn", "dev-guide"]: + if template_name not in zuul_templates: + zuul_templates.append(template_name) + else: + job_name = f"build-otc-{doc['service_type']}-{doc_type}" + if job_name not in zuul_jobs: + zuul_config_updated = True + zuul_new_jobs.append( + dict( + job=dict( + name=job_name, + parent="otc-tox-docs", + description=( + f"Build {doc_type} document using tox" + ), + files=[ + f"^{doc['rst_location']}/.*" + ], + vars=dict( + tox_envlist=doc_type + ) + ) + ) + ) + + if zuul_config_updated: + for new_job in zuul_new_jobs: + zuul_config.insert(0, new_job) + + for item in zuul_config: + if "project" in item.keys(): + project = item["project"] + project["templates"] = zuul_templates + # Ensure new jobs are in check + if len(zuul_new_jobs) > 0: + project.setdefault( + "check", + CommentedMap(jobs=[]) + ) + project["check"].yaml_set_comment_before_after_key( + key="jobs", + indent=6, + before=( + "Separate documents are rendered in check, " + "while published through regular " + "otc-tox-docs job part of the basic template" + ) + ) + project["check"]["jobs"].extend( + [x["job"]["name"] for x in zuul_new_jobs]) + + # yaml.indent(offset=2, sequence=2) + with open(zuul_file_name, "w") as f: + yaml.dump(zuul_config, f) + git_repo.index.add([zuul_file_name.name]) + if len(git_repo.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % + service["service_type"]) + return + + git_repo.index.commit( + ( + "Update zuul.yaml file\n\n" + "Performed-by: gitea/infra/otc-metadata-swiss" + "/tools/update_zuul_project_config.py" + ) + ) + push_args = ["--set-upstream", "origin", branch_name] + if args.branch_force: + push_args.append("--force") + + git_repo.git.push(*push_args) + if repo["type"] == "github": + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True + ) + elif repo["type"] == "gitea": + open_pr( + args, + repo["repo"], + dict( + title="Update zuul config", + body="Update zuul config templates", + head=branch_name, + ), + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Update zuul.yaml file in repositories." + ) + parser.add_argument( + "--environment", + required=True, + help="Repository Environment", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="zuul", + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--branch-force", + action="store_true", + help="Whether to force branch recreation.", + ) + parser.add_argument("--token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + if args.token: + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..3b5961d --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +minversion = 3.2.0 +envlist = py3,pep8 +ignore_basepython_conflict = true + +[testenv] +basepython = python3 +usedevelop = True +setenv = + PYTHONWARNINGS=default::DeprecationWarning +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run {posargs} + +[testenv:pep8] +commands = flake8 {posargs} + +[testenv:venv] +commands = {posargs} + +[flake8] +# E123, E125 skipped as they are invalid PEP-8. +show-source = True +ignore = E123,E125,W503 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build diff --git a/zuul.yaml b/zuul.yaml new file mode 100644 index 0000000..3a1ad47 --- /dev/null +++ b/zuul.yaml @@ -0,0 +1,12 @@ +--- +- project: + merge-mode: squash-merge + default-branch: main + check: + jobs: + - otc-tox-pep8 + - otc-tox-py39 + gate: + jobs: + - otc-tox-pep8 + - otc-tox-py39