commit 408b2ece8908fcb9e03991df2d45c832c33f0c8c Author: Vladimir Hasko Date: Wed May 17 14:02:14 2023 +0000 adding Swisscloud content metadata diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..485dd01 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True +source = otc-metadata-swiss + +[report] +ignore_errors = True diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4cbb844 --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +# Add patterns in here to exclude files created by tools integrated with this +# repository, such as test frameworks from the project's recommended workflow, +# rendered documentation and package builds. +# +# Don't add patterns to exclude files created by preferred personal tools +# (editors, IDEs, your operating system itself even). These should instead be +# maintained outside the repository, for example in a ~/.gitignore file added +# with: +# +# git config --global core.excludesfile '~/.gitignore' + +# Bytecompiled Python +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg* +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +cover/ +.coverage* +!.coveragerc +.tox +nosetests.xml +.testrepository +.stestr +.venv + +# Translations +*.mo + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Files created by releasenotes build +releasenotes/build diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..f497f2b --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.opendev.org +port=29418 +project=infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git.git diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..516ae6f --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# +# diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..6b9459b --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./otc_metadata/tests/ +top_dir=./ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..30439f0 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,3 @@ +The source repository for this project can be found at: + + https://github.com/infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000..6c09a78 --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,4 @@ +ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git Style Commandments +=============================================== + +Read the OpenStack Style Commandments https://docs.openstack.org/hacking/latest/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..68c771a --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..1336a36 --- /dev/null +++ b/README.rst @@ -0,0 +1,54 @@ +============ +otc-metadata +============ + +Link: ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git + + +Metadata about OTC for Ecosystem in Swisscloud environment + +Please fill here a long description which must be at least 3 lines wrapped on +80 cols, so that distribution package maintainers can use it in their packages. +Note that this is a hard requirement. + +* Free software: Apache license +* Documentation: https://docs.otc.t-systems.com/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git +* Source: https://github.com/infra/ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git + +Features +======== + +* TODO + +Overview: service.yaml +====================== + +The :code:`service.yaml` file contains all data about services, service +categories and the related documents of each service. The file is +used as a base for several internal and external applications or +websites like the Helpcenter 3.0 where the information about the document +repositories and its properties are stored. + +File structure +-------------- + +The file is based on the yaml-file format and has three main sections +which can be compared with database tables in a relational database. + +* documents: contains the information about every single document and its type + like umn, api-ref etc. + +* service category: contains the keyword and title of the service category + +* services: contains the repository information about the internal (Gitea) and + external location (GitHub) and all the necessary parameters of the service itself + +These sections, or better "tables" have +their own keys and foreign keys so that the tables are linked together and +the related information can be fetched. +For the :code:`services` table +the key is :code:`service_type` which has the foreign key in the +:code:`documents` table. So a service can have multiple documents and each +document can only be linked to one service. +The key :code:`service_category` table is :code:`name` of the service category +which is then used in the :code:`services` table as foreign key. diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..23b871f --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,4 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +otcdocstheme>=1.0.0 # Apache-2.0 +# releasenotes +reno>=3.1.0 # Apache-2.0 diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst new file mode 100644 index 0000000..1718fde --- /dev/null +++ b/doc/source/admin/index.rst @@ -0,0 +1,5 @@ +==================== +Administrators guide +==================== + +Administrators guide of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst new file mode 100644 index 0000000..7b843a2 --- /dev/null +++ b/doc/source/cli/index.rst @@ -0,0 +1,5 @@ +================================ +Command line interface reference +================================ + +CLI reference of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100755 index 0000000..9a75852 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'otcdocstheme', + #'sphinx.ext.intersphinx', +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git' +copyright = '2022, Open Telekom Cloud Developers' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] +html_theme = 'otcdocs' + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + '%s Documentation' % project, + 'Open Telekom Cloud Developers', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst new file mode 100644 index 0000000..5394ea5 --- /dev/null +++ b/doc/source/configuration/index.rst @@ -0,0 +1,5 @@ +============= +Configuration +============= + +Configuration of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/contributor/contributing.rst b/doc/source/contributor/contributing.rst new file mode 100644 index 0000000..2e91325 --- /dev/null +++ b/doc/source/contributor/contributing.rst @@ -0,0 +1,5 @@ +============================ +So You Want to Contribute... +============================ + +TODO diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 0000000..036e449 --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1,9 @@ +=========================== + Contributor Documentation +=========================== + +.. toctree:: + :maxdepth: 2 + + contributing + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..6a96f30 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,30 @@ +.. ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +============================================ +Welcome to the documentation of otc-metadata +============================================ + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + install/index + library/index + contributor/index + configuration/index + cli/index + user/index + admin/index + reference/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst new file mode 100644 index 0000000..6d7dc44 --- /dev/null +++ b/doc/source/install/index.rst @@ -0,0 +1,14 @@ +================================================ + {{cookiecutter.module_name}} installation guide +================================================ + +.. toctree:: + :maxdepth: 2 + + install.rst + +The otc-metadata provides... + +This chapter assumes a working setup of OpenStack following the +`OpenStack Installation Tutorial +`_. diff --git a/doc/source/install/install.rst b/doc/source/install/install.rst new file mode 100644 index 0000000..3d5c5a7 --- /dev/null +++ b/doc/source/install/install.rst @@ -0,0 +1,13 @@ +.. _install: + +Install and configure +~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the +otc-metadata. + +This section assumes that you already have a working OpenStack +environment with at least the following components installed: +.. (add the appropriate services here and further notes) + +Note that installation and configuration vary by distribution. diff --git a/doc/source/library/index.rst b/doc/source/library/index.rst new file mode 100644 index 0000000..e884b9b --- /dev/null +++ b/doc/source/library/index.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git in a project:: + + import otc-metadata diff --git a/doc/source/readme.rst b/doc/source/readme.rst new file mode 100644 index 0000000..a6210d3 --- /dev/null +++ b/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst new file mode 100644 index 0000000..1d57317 --- /dev/null +++ b/doc/source/reference/index.rst @@ -0,0 +1,5 @@ +========== +References +========== + +References of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..efbc805 --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,5 @@ +=========== +Users guide +=========== + +Users guide of ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/infra/otc-metadata-swiss.git. diff --git a/otc_metadata/__init__.py b/otc_metadata/__init__.py new file mode 100644 index 0000000..56b98c7 --- /dev/null +++ b/otc_metadata/__init__.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +__all__ = ['__version__', 'Docs'] + +import pbr.version + +from otc_metadata.services import Services # flake8: noqa + +__version__ = pbr.version.VersionInfo('otc-metadata').version_string() +_service_manager = None + + +def get_service_data(*args, **kwargs): + """Return singleton instance of the Services object. + Parameters are all passed through to the + :class:`~otc_metadata.services.Services` constructor. + .. note:: + Only one singleton is kept, so if instances with different parameter + values are desired, directly calling the constructor is necessary. + :returns: Singleton instance of + :class:`~otc_metadata.services.Services` + """ + global _service_manager + if not _service_manager: + _service_manager = Services(*args, **kwargs) + return _service_manager diff --git a/otc_metadata/data/__init__.py b/otc_metadata/data/__init__.py new file mode 100644 index 0000000..2102a7e --- /dev/null +++ b/otc_metadata/data/__init__.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml + +__all__ = ['read_data'] + +DATA_DIR = os.path.dirname(__file__) + + +def read_data(filename): + """Return data that is shipped inside the Python package. + + """ + + filepath = os.path.join(DATA_DIR, filename) + with open(filepath, 'r') as fd: + return yaml.safe_load(fd) + + +def rewrite_data(filename, data): + """Rewrites data formatting it + + """ + from ruamel.yaml import YAML + _yaml = YAML() + _yaml.indent(mapping=2, sequence=4, offset=2) + filepath = os.path.join(DATA_DIR, filename) + with open(filepath, 'w') as fd: + _yaml.dump(data, fd) diff --git a/otc_metadata/data/services.yaml b/otc_metadata/data/services.yaml new file mode 100644 index 0000000..0323254 --- /dev/null +++ b/otc_metadata/data/services.yaml @@ -0,0 +1,985 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Please consult with otc_metadata.services.Services:_sort_data for expected +# sort order. When unsure perform tools/sort_metadata.py for getting it sorted. +--- +documents: + - hc_location: api/aom + html_location: docs/aom/api-ref + link: /application-operations-management/api-ref/ + pdf_name: aom-api-ref + rst_location: api-ref/source + service_type: aom + title: API Reference + type: api-ref + - hc_location: usermanual/aom + html_location: docs/aom/umn + link: /application-operations-management/umn/ + pdf_name: aom-umn + rst_location: umn/source + service_type: aom + title: User Guide + type: umn + - hc_location: api/as + html_location: docs/as/api-ref + link: /auto-scaling/api-ref/ + pdf_name: as-api-ref + rst_location: api-ref/source + service_type: as + title: API Reference + type: api-ref + - hc_location: devg/as + html_location: docs/as/dev + link: /auto-scaling/dev-guide/ + pdf_name: as-dev-guide + rst_location: dev_guide/source + service_type: as + title: Developer Guide + type: dev + - hc_location: usermanual/as + html_location: docs/as/umn + link: /auto-scaling/umn/ + pdf_name: as-umn + rst_location: umn/source + service_type: as + title: User Guide + type: umn + - hc_location: api/cbr + html_location: docs/cbr/api-ref + link: /cloud-backup-recovery/api-ref/ + pdf_name: cbr-api-ref + rst_location: api-ref/source + service_type: cbr + title: API Reference + type: api-ref + - hc_location: usermanual/cbr + html_location: docs/cbr/umn + link: /cloud-backup-recovery/umn/ + pdf_name: cbr-umn + rst_location: umn/source + service_type: cbr + title: User Guide + type: umn + - hc_location: api2/cce + html_location: docs/cce/api-ref + link: /cloud-container-engine/api-ref/ + pdf_name: cce-api-ref + rst_location: api-ref/source + service_type: cce + title: API Reference + type: api-ref + - hc_location: usermanual2/cce + html_location: docs/cce/umn + link: /cloud-container-engine/umn/ + rst_location: umn/source + service_type: cce + title: User Guide + type: umn + - hc_location: api/ces + html_location: docs/ces/api-ref + link: /cloud-eye/api-ref/ + pdf_name: ces-api-ref + rst_location: api-ref/source + service_type: ces + title: API Reference + type: api-ref + - hc_location: usermanual/ces + html_location: docs/ces/umn + link: /cloud-eye/umn/ + pdf_name: ces-umn + rst_location: umn/source + service_type: ces + title: User Guide + type: umn + - hc_location: api/cts + html_location: docs/cts/api-ref + link: /cloud-trace-service/api-ref/ + pdf_name: cts-api-ref + rst_location: api-ref/source + service_type: cts + title: API Reference + type: api-ref + - hc_location: usermanual/cts + html_location: docs/cts/umn + link: /cloud-trace-service/umn/ + pdf_name: cts-umn + rst_location: umn/source + service_type: cts + title: User Guide + type: umn + - hc_location: api/dc + html_location: docs/dc/api-ref + link: /direct-connect/api-ref/ + pdf_name: dc-api-ref + rst_location: api-ref/source + service_type: dc + title: API Reference + type: api-ref + - hc_location: usermanual/dc + html_location: docs/dc/umn + link: /direct-connect/umn/ + pdf_name: dc-umn + rst_location: umn/source + service_type: dc + title: User Guide + type: umn + - hc_location: api/deh + html_location: docs/deh/api-ref + link: /dedicated-host/api-ref/ + pdf_name: deh-api-ref + rst_location: api-ref/source + service_type: deh + title: API Reference + type: api-ref + - hc_location: usermanual/deh + html_location: docs/deh/umn + link: /dedicated-host/umn/ + pdf_name: deh-umn + rst_location: umn/source + service_type: deh + title: User Guide + type: umn + - hc_location: api/dns + html_location: docs/dns/api-ref + link: /domain-name-service/api-ref/ + pdf_name: dns-api-ref + rst_location: api-ref/source + service_type: dns + title: API Reference + type: api-ref + - hc_location: usermanual/dns + html_location: docs/dns/umn + link: /domain-name-service/umn/ + pdf_name: dns-umn + rst_location: umn/source + service_type: dns + title: User Guide + type: umn + - hc_location: api/ecs + html_location: docs/ecs/api-ref + link: /elastic-cloud-server/api-ref/ + pdf_name: ecs-api-ref + rst_location: api-ref/source + service_type: ecs + title: API Reference + type: api-ref + - hc_location: devg/ecs + html_location: docs/ecs/dev + link: /elastic-cloud-server/dev-guide/ + pdf_name: ecs-dev-guide + rst_location: dev_guide/source + service_type: ecs + title: Developer Guide + type: dev + - hc_location: usermanual/ecs + html_location: docs/ecs/umn + link: /elastic-cloud-server/umn/ + pdf_name: ecs-umn + rst_location: umn/source + service_type: ecs + title: User Guide + type: umn + - hc_location: api/eip + html_location: docs/eip/api-ref + link: /elastic-ip/api-ref/ + pdf_name: eip-api-ref + rst_location: api-ref/source + service_type: eip + title: API Reference + type: api-ref + - hc_location: usermanual/eip + html_location: docs/eip/umn + link: /elastic-ip/umn/ + pdf_name: eip-umn + rst_location: umn/source + service_type: eip + title: User Guide + type: umn + - hc_location: api/elb + html_location: docs/elb/api-ref + link: /elastic-load-balancing/api-ref/ + pdf_name: elb-api-ref + rst_location: api-ref/source + service_type: elb + title: API Reference + type: api-ref + - hc_location: usermanual/elb + html_location: docs/elb/umn + link: /elastic-load-balancing/umn/ + pdf_name: elb-umn + rst_location: umn/source + service_type: elb + title: User Guide + type: umn + - hc_location: api/evs + html_location: docs/evs/api-ref + link: /elastic-volume-service/api-ref/ + pdf_name: evs-api-ref + rst_location: api-ref/source + service_type: evs + title: API Reference + type: api-ref + - hc_location: devg/evs + html_location: docs/evs/dev + link: /elastic-volume-service/dev-guide/ + pdf_name: evs-dev-guide + rst_location: dev_guide/source + service_type: evs + title: Developer Guide + type: dev + - hc_location: usermanual/evs + html_location: docs/evs/umn + link: /elastic-volume-service/umn/ + pdf_name: evs-umn + rst_location: umn/source + service_type: evs + title: User Guide + type: umn + - hc_location: api/iam + html_location: docs/iam/api-ref + link: /identity-access-management/api-ref/ + pdf_name: iam-api-ref + rst_location: api-ref/source + service_type: iam + title: API Reference + type: api-ref + - hc_location: usermanual/iam + html_location: docs/iam/umn + link: /identity-access-management/umn/ + pdf_name: iam-umn + rst_location: umn/source + service_type: iam + title: User Guide + type: umn + - hc_location: api/ims + html_location: docs/ims/api-ref + link: /image-management-service/api-ref/ + pdf_name: ims-api-ref + rst_location: api-ref/source + service_type: ims + title: API Reference + type: api-ref + - hc_location: devg/ims + html_location: docs/ims/dev + link: /image-management-service/dev-guide/ + pdf_name: ims-dev-guide + rst_location: dev_guide/source + service_type: ims + title: Developer Guide + type: dev + - html_location: docs/ims/public-images + link: /image-management-service/public-images/ + pdf_name: ims-public-images + rst_location: doc/public-images/source + service_type: ims + title: Public Image Introduction + type: public-images + - hc_location: usermanual/ims + html_location: docs/ims/umn + link: /image-management-service/umn/ + pdf_name: ims-umn + rst_location: umn/source + service_type: ims + title: User Guide + type: umn + - hc_location: api/kms + html_location: docs/kms/api-ref + link: /key-management-service/api-ref/ + pdf_name: kms-api-ref + rst_location: api-ref/source + service_type: kms + title: API Reference + type: api-ref + - hc_location: usermanual/kms + html_location: docs/kms/umn + link: /key-management-service/umn/ + pdf_name: kms-umn + rst_location: umn/source + service_type: kms + title: User Guide + type: umn + - hc_location: api/nat + html_location: docs/natgw/api-ref + link: /nat-gateway/api-ref/ + pdf_name: natgw-api-ref + rst_location: api-ref/source + service_type: natgw + title: API Reference + type: api-ref + - hc_location: usermanual/nat + html_location: docs/natgw/umn + link: /nat-gateway/umn/ + pdf_name: natgw-umn + rst_location: umn/source + service_type: natgw + title: User Guide + type: umn + - hc_location: api/obs + html_location: docs/obs/api-ref + link: /object-storage-service/api-ref/ + pdf_name: obs-api-ref + rst_location: api-ref/source + service_type: obs + title: API Reference + type: api-ref + - hc_location: browsertg/obs + html_location: docs/obs/tool + link: /object-storage-service/tool-guide/ + pdf_name: obs-tool-guide + rst_location: doc/tool/source + service_type: obs + title: Tool Guide (OBS Browser) + type: tool-guide + - hc_location: pfs/obs + html_location: docs/obs/pfs + link: /object-storage-service/parallel-file-system/ + pdf_name: obs-pfs + rst_location: doc/parallel-file-system/source + service_type: obs + title: Parallel File System (PFS) + type: parallel-file-system + - html_location: docs/obs/perms-cfg + link: /object-storage-service/permissions-configuration-guide/ + pdf_name: obs-perms-cfg + rst_location: doc/permissions-configuration-guide/source + service_type: obs + title: Permissions Configuration Guide + type: permissions-configuration-guide + - hc_location: api_swift/obs + html_location: docs/obs/api-swift + link: /object-storage-service/swiftapi/ + pdf_name: obs-swiftapi + rst_location: doc/swiftapi/source + service_type: obs + title: API Reference (Swift) + type: swiftapi + - hc_location: api_obs/obs + html_location: docs/obs/s3api + link: /object-storage-service/s3api/ + pdf_name: obs-s3api + rst_location: doc/s3api/source + service_type: obs + title: S3 API Reference + type: s3api + - hc_location: usermanual/obs + html_location: docs/obs/umn + link: /object-storage-service/umn/ + pdf_name: obs-umn + rst_location: umn/source + service_type: obs + title: User Guide + type: umn + - hc_location: api/rds + html_location: docs/rds/api-ref + link: /relational-database-service/api-ref/ + pdf_name: rds-api-ref + rst_location: api-ref/source + service_type: rds + title: API Reference + type: api-ref + - hc_location: usermanual/rds + html_location: docs/rds/umn + link: /relational-database-service/umn/ + pdf_name: rds-umn + rst_location: umn/source + service_type: rds + title: User Guide + type: umn + - hc_location: usermanual/sd + html_location: docs/sd/umn + link: /status-dashboard/umn/ + pdf_name: sd-umn + rst_location: umn/source + service_type: sd + title: User Guide + type: umn + - hc_location: api/sdrs + html_location: docs/sdrs/api-ref + link: /storage-disaster-recovery-service/api-ref/ + pdf_name: sdrs-api-ref + rst_location: api-ref/source + service_type: sdrs + title: API Reference + type: api-ref + - hc_location: usermanual/sdrs + html_location: docs/sdrs/umn + link: /storage-disaster-recovery-service/umn/ + pdf_name: sdrs-umn + rst_location: umn/source + service_type: sdrs + title: User Guide + type: umn + - hc_location: api/smn + html_location: docs/smn/api-ref + link: /simple-message-notification/api-ref/ + pdf_name: smn-api-ref + rst_location: api-ref/source + service_type: smn + title: API Reference + type: api-ref + - hc_location: usermanual/smn + html_location: docs/smn/umn + link: /simple-message-notification/umn/ + pdf_name: smn-umn + rst_location: umn/source + service_type: smn + title: User Guide + type: umn + - hc_location: api/swr + html_location: docs/swr/api-ref + link: /software-repository-container/api-ref/ + pdf_name: swr-api-ref + rst_location: api-ref/source + service_type: swr + title: API Reference + type: api-ref + - hc_location: usermanual/swr + html_location: docs/swr/umn + link: /software-repository-container/umn/ + pdf_name: swr-umn + rst_location: umn/source + service_type: swr + title: User Guide + type: umn + - hc_location: api/tms + html_location: docs/tms/api-ref + link: /tag-management-service/api-ref/ + pdf_name: tms-api-ref + rst_location: api-ref/source + service_type: tms + title: API Reference + type: api-ref + - hc_location: usermanual/tms + html_location: docs/tms/umn + link: /tag-management-service/umn/ + pdf_name: tms-umn + rst_location: umn/source + service_type: tms + title: User Guide + type: umn + - hc_location: api/vpc + html_location: docs/vpc/api-ref + link: /virtual-private-cloud/api-ref/ + pdf_name: vpc-api-ref + rst_location: api-ref/source + service_type: vpc + title: API Reference + type: api-ref + - hc_location: usermanual/vpc + html_location: docs/vpc/umn + link: /virtual-private-cloud/umn/ + pdf_name: vpc-umn + rst_location: umn/source + service_type: vpc + title: User Guide + type: umn + - hc_location: api/vpcep + html_location: docs/vpcep/api-ref + link: /vpc-endpoint/api-ref/ + pdf_name: vpcep-api-ref + rst_location: api-ref/source + service_type: vpcep + title: API Reference + type: api-ref + - hc_location: usermanual/vpcep + html_location: docs/vpcep/umn + link: /vpc-endpoint/umn/ + pdf_name: vpcep-umn + rst_location: umn/source + service_type: vpcep + title: User Guide + type: umn + - hc_location: api/vpn + html_location: docs/vpn/api-ref + link: /virtual-private-network/api-ref/ + pdf_name: vpn-api-ref + rst_location: api-ref/source + service_type: vpn + title: API Reference + type: api-ref + - hc_location: usermanual/vpn + html_location: docs/vpn/umn + link: /virtual-private-network/umn/ + pdf_name: vpn-umn + rst_location: umn/source + service_type: vpn + title: User Guide + type: umn + - environment: internal + hc_location: api/dwaf + html_location: docs/wafd/api-ref + link: /web-application-firewall-dedicated/api-ref/ + pdf_name: wafd-api-ref + rst_location: api-ref/source + service_type: wafd + title: API Reference + type: api-ref + - environment: internal + hc_location: usermanual/dwaf + html_location: docs/wafd/umn + link: /web-application-firewall-dedicated/umn/ + pdf_name: wafd-umn + rst_location: umn/source + service_type: wafd + title: User Guide + type: umn +service_categories: + - name: application + title: Application Services + - name: big_data + title: Big Data and Data Analysis + - name: compute + title: Compute + - name: container + title: Container + - name: database + title: Database + - name: network + title: Networking + - name: md + title: Management & Deployment + - name: security-services + title: Security Services + - name: storage + title: Storage + - name: other + title: Other +services: + - repositories: + - environment: internal + repo: docs/application-operations-management + type: gitea + - environment: public + repo: opentelekomcloud-docs/application-operations-management + type: github + service_category: application + service_title: Application Operations Management + service_type: aom + service_uri: application-operations-management + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/auto-scaling + type: gitea + - environment: public + repo: opentelekomcloud-docs/auto-scaling + type: github + service_category: compute + service_title: Auto Scaling + service_type: as + service_uri: auto-scaling + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-backup-recovery + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-backup-recovery + type: github + service_category: storage + service_title: Cloud Backup and Recovery + service_type: cbr + service_uri: cloud-backup-recovery + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-container-engine + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-container-engine + type: github + service_category: container + service_title: Cloud Container Engine + service_type: cce + service_uri: cloud-container-engine + environment: public + teams: + - name: docs-container-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-eye + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-eye + type: github + service_category: md + service_title: Cloud Eye + service_type: ces + service_uri: cloud-eye + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/cloud-trace-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/cloud-trace-service + type: github + service_category: md + service_title: Cloud Trace Service + service_type: cts + service_uri: cloud-trace-service + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/direct-connect + type: gitea + - environment: public + repo: opentelekomcloud-docs/direct-connect + type: github + service_category: network + service_title: Direct Connect + service_type: dc + service_uri: direct-connect + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/dedicated-host + type: gitea + - environment: public + repo: opentelekomcloud-docs/dedicated-host + type: github + service_category: compute + service_title: Dedicated Host + service_type: deh + service_uri: dedicated-host + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/domain-name-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/domain-name-service + type: github + service_category: network + service_title: Domain Name Service + service_type: dns + service_uri: domain-name-service + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - repo: docs/elastic-cloud-server + type: gitea + environment: internal + teams: + - name: docs-compute-ro + permission: read + - name: docs-compute-rw + permission: write + assignees: + - docs-compute-ro + - repo: opentelekomcloud-docs/elastic-cloud-server + type: github + environment: public + service_category: compute + service_title: Elastic Cloud Server + service_type: ecs + service_uri: elastic-cloud-server + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/elastic-ip + type: gitea + - environment: public + repo: opentelekomcloud-docs/elastic-ip + type: github + service_category: network + service_title: Elastic IP + service_type: eip + service_uri: elastic-ip + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/elastic-load-balancing + type: gitea + - environment: public + repo: opentelekomcloud-docs/elastic-load-balancing + type: github + service_category: network + service_title: Elastic Load Balancing + service_type: elb + service_uri: elastic-load-balancing + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/elastic-volume-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/elastic-volume-service + type: github + service_category: storage + service_title: Elastic Volume Service + service_type: evs + service_uri: elastic-volume-service + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/identity-access-management + type: gitea + - environment: public + repo: opentelekomcloud-docs/identity-access-management + type: github + service_category: security-services + service_title: Identity and Access Management + service_type: iam + service_uri: identity-access-management + environment: public + teams: + - name: docs-security-services-rw + permission: write + - repositories: + - environment: internal + repo: docs/image-management-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/image-management-service + type: github + service_category: compute + service_title: Image Management Service + service_type: ims + service_uri: image-management-service + environment: public + teams: + - name: docs-compute-rw + permission: write + - repositories: + - environment: internal + repo: docs/key-management-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/key-management-service + type: github + service_category: security-services + service_title: Key Management Service + service_type: kms + service_uri: key-management-service + environment: public + teams: + - name: docs-security-services-rw + permission: write + - repositories: + - environment: internal + repo: docs/nat-gateway + type: gitea + - environment: public + repo: opentelekomcloud-docs/nat-gateway + type: github + service_category: network + service_title: NAT Gateway + service_type: natgw + service_uri: nat-gateway + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/object-storage-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/object-storage-service + type: github + service_category: storage + service_title: Object Storage Service + service_type: obs + service_uri: object-storage-service + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/relational-database-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/relational-database-service + type: github + service_category: database + service_title: Relational Database Service + service_type: rds + service_uri: relational-database-service + environment: public + teams: + - name: docs-database-rw + permission: write + - repositories: + - environment: internal + repo: docs/status-dashboard + type: gitea + - environment: public + repo: opentelekomcloud-docs/status-dashboard + type: github + service_category: other + service_title: Status Dashboard + service_type: sd + service_uri: status-dashboard/umn + environment: hidden + teams: + - name: docs-eco-rw + permission: write + - repositories: + - environment: internal + repo: docs/storage-disaster-recovery-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/storage-disaster-recovery-service + type: github + service_category: storage + service_title: Storage Disaster Recovery Service + service_type: sdrs + service_uri: storage-disaster-recovery-service + environment: public + teams: + - name: docs-storage-rw + permission: write + - repositories: + - environment: internal + repo: docs/simple-message-notification + type: gitea + - environment: public + repo: opentelekomcloud-docs/simple-message-notification + type: github + service_category: application + service_title: Simple Message Notification + service_type: smn + service_uri: simple-message-notification + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/software-repository-container + type: gitea + - environment: public + repo: opentelekomcloud-docs/software-repository-container + type: github + service_category: container + service_title: Software Repository for Containers + service_type: swr + service_uri: software-repository-container + environment: public + teams: + - name: docs-container-rw + permission: write + - repositories: + - environment: internal + repo: docs/tag-management-service + type: gitea + - environment: public + repo: opentelekomcloud-docs/tag-management-service + type: github + service_category: md + service_title: Tag Management Service + service_type: tms + service_uri: tag-management-service + environment: public + teams: + - name: docs-orchestration-rw + permission: write + - repositories: + - environment: internal + repo: docs/virtual-private-cloud + type: gitea + - environment: public + repo: opentelekomcloud-docs/virtual-private-cloud + type: github + service_category: network + service_title: Virtual Private Cloud + service_type: vpc + service_uri: virtual-private-cloud + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/vpc-endpoint + type: gitea + - environment: public + repo: opentelekomcloud-docs/vpc-endpoint + type: github + service_category: network + service_title: VPC Endpoint + service_type: vpcep + service_uri: vpc-endpoint + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/virtual-private-network + type: gitea + - environment: public + repo: opentelekomcloud-docs/virtual-private-network + type: github + service_category: network + service_title: Virtual Private Network + service_type: vpn + service_uri: virtual-private-network + environment: public + teams: + - name: docs-network-rw + permission: write + - repositories: + - environment: internal + repo: docs/web-application-firewall-dedicated + type: gitea + - environment: public + repo: opentelekomcloud-docs/web-application-firewall-dedicated + type: github + service_category: security-services + service_title: Dedicated Web Application Firewall + service_type: wafd + service_uri: web-application-firewall-dedicated + environment: public + teams: + - name: docs-security-services-rw + permission: write diff --git a/otc_metadata/docs.py b/otc_metadata/docs.py new file mode 100644 index 0000000..733add1 --- /dev/null +++ b/otc_metadata/docs.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +import otc_metadata.data + +__all__ = ['Service'] + +BUILTIN_DATA = otc_metadata.data.read_data('docs.yaml') + + +def _normalize_type(service_type): + if service_type: + return service_type.replace('_', '-') + + +class Service(object): + """Encapsulation of the OTC Docs data + """ + + def __init__(self): + self._service_data = BUILTIN_DATA + + @property + def all_services(self): + "Service Categories data listing." + return copy.deepcopy(self._service_data['services']) diff --git a/otc_metadata/services.py b/otc_metadata/services.py new file mode 100644 index 0000000..195b067 --- /dev/null +++ b/otc_metadata/services.py @@ -0,0 +1,230 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import warnings + +import otc_metadata.data + +__all__ = ["Services"] + +BUILTIN_DATA = otc_metadata.data.read_data("services.yaml") + + +def _normalize_type(service_type): + if service_type: + return service_type.replace("_", "-") + + +class Services(object): + """Encapsulation of the OTC Services data""" + + def __init__(self): + self._service_data = BUILTIN_DATA + + def _sort_data(self): + # Sort every doc item by key + sorted_docs = [] + for doc in self._service_data["documents"]: + sorted_docs.append(dict(sorted(doc.items(), key=lambda kv: kv[0]))) + + # sort docs list by _ + self._service_data["documents"] = sorted( + sorted_docs, + key=lambda x: f"{x.get('service_type')}{x.get('title')}" + ) + # sort services by <service_type>_<service_title> + self._service_data["services"] = sorted( + self._service_data["services"], + key=lambda x: f"{x.get('service_type')}{x.get('service_title')}", + ) + + def _rewrite_data(self): + otc_metadata.data.rewrite_data("services.yaml", self._service_data) + + @property + def all_services(self): + "Service Categories data listing." + return copy.deepcopy(self._service_data["services"]) + + @property + def all_docs(self): + "Service Docs data listing." + return copy.deepcopy(self._service_data["documents"]) + + @property + def service_dict(self): + "Service Docs data listing." + res = dict() + for srv in self.all_services: + res[srv["service_type"]] = copy.deepcopy(srv) + return res + + @property + def service_categories(self): + """List services categories""" + res = [] + for cat in self._service_data["service_categories"]: + res.append(copy.deepcopy(cat)) + return res + + def services_by_category(self, category): + """List services matching category""" + res = [] + for srv in self.all_services: + if srv["service_category"] == category: + res.append(copy.deepcopy(srv)) + return res + + def services_with_docs_by_category(self, category, environment=None): + """Retrieve service category docs data + + :param str category: Optional Category filter + :param str env: Optional service environment. Influeces "repository" + field + """ + res = dict() + services = self.service_dict + for doc in self.all_docs: + cat = doc["service_type"] + service = services.get(cat) + if not service: + warnings.warn("No Service defition of type %s" % (cat)) + continue + if category and service["service_category"] != category: + continue + res.setdefault(cat, service) + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + if "environment" in doc and doc["environment"] != environment: + continue + res[cat].setdefault("docs", []) + res[cat]["docs"].append(res_doc) + return res + + def docs_by_service_category(self, category, environment=None): + """List services matching category + + :param str category: Category name + :param str env: Optional service environment. Influeces "repository" + field + """ + res = [] + services = self.service_dict + for doc in self.all_docs: + cat = doc["service_type"] + service = services.get(cat) + if not service: + warnings.warn("No Service defition of type %s" % (cat)) + continue + if service["service_category"] == category: + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + for srv_env in service["repositories"]: + if srv_env.get("environment") == environment: + res_doc["repository"] = srv_env["repo"] + res.append(res_doc) + return res + + def docs_by_service_type(self, service_type): + """List documents of the service + + :param str service_type: Service type + :returns: generator for documents + """ + for doc in self.all_docs: + if doc["service_type"] != service_type: + continue + yield copy.deepcopy(doc) + + def all_docs_full(self, environment): + """Return list or documents with full service data""" + services = self.service_dict + for doc in self.all_docs: + if not doc["service_type"] in services: + print(f"No service type {doc['service_type']}") + continue + service = services[doc["service_type"]] + res_doc = copy.deepcopy(doc) + res_doc.update(**service) + if environment: + for srv_env in service["repositories"]: + if srv_env.get("environment") == environment: + res_doc["repository"] = srv_env["repo"] + for srv_assignees in service.get("assignees", []): + if srv_assignees.get("environment") == environment: + res_doc["assignees"] = srv_assignees["names"] + yield res_doc + + def docs_html_by_category(self, environment): + """Generate structure for doc-exports repository""" + doc_struct = dict() + for srv in self.all_services: + doc_struct.setdefault(srv["service_category"], []) + srv_res = dict( + service_title=srv["service_title"], + service_type=srv["service_type"], + docs=[], + ) + if "teams" in srv: + srv_res["teams"] = [ + x for x in srv["teams"] if x["permission"] == "write" + ] + if "repositories" in srv and environment: + for repo in srv["repositories"]: + if ( + "environment" in repo + and repo["environment"] == environment + ): + srv_res["repository"] = repo["repo"] + for doc in self.all_docs: + if ( + "html_location" in doc + and doc["service_type"] == srv_res["service_type"] + ): + doc_res = dict( + html_location=doc["html_location"], + rst_location=doc["rst_location"], + title=doc["title"], + type=doc.get("type", "dummy"), + ) + if "pdf_name" in doc: + doc_res["pdf_name"] = doc["pdf_name"] + if "hc_location" in doc: + doc_res["hc_location"] = doc["hc_location"] + srv_res["docs"].append(doc_res) + if len(srv_res["docs"]) > 0: + doc_struct[srv["service_category"]].append(srv_res) + + return dict(categories=doc_struct) + + def get_service_with_docs_by_service_type(self, service_type): + """Retrieve service and service docs by service_type + + :param str service_type: Filter by service_type + """ + res = dict() + res['service'] = {} + docs = [] + services = self._service_data + for doc in services['documents']: + if doc['service_type'] == service_type: + docs.append(doc) + res['documents'] = docs + for service in services['services']: + if service['service_type'] == service_type: + res['service'] = service + break + return res diff --git a/otc_metadata/templates/conf.py.j2 b/otc_metadata/templates/conf.py.j2 new file mode 100644 index 0000000..3228eec --- /dev/null +++ b/otc_metadata/templates/conf.py.j2 @@ -0,0 +1,122 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# !!! +# This file is generated out of template in doc-exports repository. +# Beware overwriting it locally. + +import os +import sys + +extensions = [ + 'otcdocstheme', +{%- if otc_sbv %} + 'otc_sphinx_directives' +{%- endif %} +] + +otcdocs_auto_name = False +otcdocs_auto_version = False + +project = '{{ project }}' +otcdocs_repo_name = '{{ repo_name }}' +# Those variables are required for edit/bug links +{%- if git_fqdn %} +otcdocs_git_fqdn = '{{ git_fqdn }}' +{%- endif %} +{%- if git_type %} +otcdocs_git_type = '{{ git_type }}' +{%- endif %} + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) + +# -- General configuration ---------------------------------------------------- +# https://docutils.sourceforge.io/docs/user/smartquotes.html - it does not +# what it is expected +smartquotes = False + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +copyright = u'2022-present, Open Telekom Cloud' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +language = 'en' + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { +{%- if html_options is defined -%} +{%- for (k, v) in html_options.items() %} +{%- if v is boolean %} + "{{ k }}": {{ v }}, +{%- else %} + "{{ k }}": "{{ v }}", +{%- endif %} +{%- endfor %} +{%- endif %} +} + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +{% if title is defined %} +html_title = "{{ title }}" +{% endif %} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Do not include sources into the rendered results +html_copy_source = False + +# -- Options for PDF output -------------------------------------------------- +latex_documents = [ +{%- if pdf_name is defined %} + ('index', + '{{ pdf_name }}.tex', + u'{{ title }}', + u'OpenTelekomCloud', 'manual'), +{% endif -%} +] + diff --git a/otc_metadata/templates/doc_requirements.txt.j2 b/otc_metadata/templates/doc_requirements.txt.j2 new file mode 100644 index 0000000..87daf92 --- /dev/null +++ b/otc_metadata/templates/doc_requirements.txt.j2 @@ -0,0 +1,7 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +otcdocstheme # Apache-2.0 +# releasenotes +reno>=3.1.0 # Apache-2.0 + +otc-sphinx-directives>=0.1.0 +git+https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-swiss.git#egg=otc_metadata diff --git a/otc_metadata/templates/index_sbv.rst.j2 b/otc_metadata/templates/index_sbv.rst.j2 new file mode 100644 index 0000000..1ef5b68 --- /dev/null +++ b/otc_metadata/templates/index_sbv.rst.j2 @@ -0,0 +1,7 @@ +{{ sbv_title }} + +.. directive_wrapper:: + :class: container-sbv + + .. service_card:: + :service_type: {{ service_type }} diff --git a/otc_metadata/templates/tox.ini.j2 b/otc_metadata/templates/tox.ini.j2 new file mode 100644 index 0000000..e6ecf9d --- /dev/null +++ b/otc_metadata/templates/tox.ini.j2 @@ -0,0 +1,125 @@ +[tox] +minversion = 3.1 +envlist = py39,pep8 +skipsdist = True +ignore_basepython_conflict = True + +[testenv] +usedevelop = True +install_command = pip install {opts} {packages} +deps = + -r{toxinidir}/requirements.txt +commands = stestr run {posargs} + stestr slowest + +[testenv:pep8] +commands = + doc8 doc/source README.rst + +[testenv:venv] +deps = + -r{toxinidir}/requirements.txt +commands = {posargs} + +# This env is invoked in the periodic pipeline and is therefore responsible to +# build all relevant docs at once. +[testenv:docs] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + mkdir + cp + sh + rm + sphinx-build +commands = + rm -rf doc/build/html doc/build/doctrees + sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html +{%- for doc in docs %} + {[testenv:{{ doc.type }}]commands} + {[testenv:json-{{ doc.type }}]commands} +{%- endfor %} + +[testenv:docs-pdf] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + rm + mkdir + make + bash + cp +commands = + mkdir -p doc/build/pdf +{%- for doc in docs %} +{%- if doc.pdf_name %} + {[testenv:{{ doc.type }}-pdf-docs]commands} +{%- endif %} +{%- endfor %} + +{% for doc in docs -%} +{% set loc = doc.rst_location | replace('/source', '') %} +# HTML version +[testenv:{{ doc.type }}] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + cp + mkdir +commands = + sphinx-build -W --keep-going -b html {{ loc }}/source doc/build/html/{{ doc.type }} +{%- if doc.type == 'api-ref' %} + mkdir -p api-ref/build/html + cp -av doc/build/html/api-ref api-ref/build/html +{%- elif doc.type == 'umn' %} + mkdir -p umn/build/html + cp -av doc/build/html/umn umn/build/html +{%- elif doc.type == 'dev-guide' %} + mkdir -p dev_guide/build/html + cp -av doc/build/html/dev-guide dev_guide/build/html +{%- endif %} + +# Json version (for search) +[testenv:json-{{ doc.type }}] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + cp + mkdir + sh +commands = + sphinx-build -W --keep-going -b json {{ loc }}/source doc/build/json/{{ doc.type }} + # Drop data useless for the search - wrap it also with sh/xargs due to bugs + # in tox + sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf" +{%- if doc.type == 'api-ref' %} + mkdir -p api-ref/build/json + cp -av doc/build/json/api-ref api-ref/build/json +{%- elif doc.type == 'umn' %} + mkdir -p umn/build/json + cp -av doc/build/json/umn umn/build/json +{%- elif doc.type == 'dev-guide' %} + mkdir -p dev_guide/build/json + cp -av doc/build/json/dev-guide dev_guide/build/json +{%- endif %} + +{% if doc.pdf_name -%} +# PDF version +[testenv:{{ doc.type }}-pdf-docs] +deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + rm + mkdir + make + bash + cp +commands = + rm -rf {{ loc }}/build/pdf + sphinx-build -W --keep-going -b latex {{ loc }}/source {{ loc }}/build/pdf/ + bash -c "for f in {{ loc }}/build/pdf/*.gif; do convert $f $\{f/%gif/png\}; done || true" + bash -c "for f in {{ loc }}/build/pdf/*.tex; do sed -iorig 's/\.gif//g' $f; done" + make -C {{ loc }}/build/pdf + mkdir -p doc/build/pdf + cp {{ loc }}/build/pdf/{{ doc.pdf_name }}.pdf doc/build/pdf/ +{% endif %} +{% endfor %} + +[doc8] +ignore = D001 +extensions = .rst, .yaml \ No newline at end of file diff --git a/otc_metadata/tests/__init__.py b/otc_metadata/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/otc_metadata/tests/test_otc_metadata.py b/otc_metadata/tests/test_otc_metadata.py new file mode 100644 index 0000000..609199f --- /dev/null +++ b/otc_metadata/tests/test_otc_metadata.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +test_otc-metadata +---------------------------------- + +Tests for `otc-metadata` module. +""" + +from unittest import TestCase + +from otc_metadata import services + + +class TestOtcMetadata(TestCase): + def setUp(self): + self.data = services.Services() + + def test_data_is_sorted(self): + curr = self.data + new = services.Services() + new._sort_data() + self.assertEqual( + curr._service_data, new._service_data, "Data is sorted properly" + ) + + def test_service_categories(self): + category = dict() + for cat in self.data._service_data["service_categories"]: + category[cat["name"]] = cat["title"] + for srv in self.data.all_services: + self.assertTrue( + srv["service_category"] in category, + f"Category {srv['service_category']} is present", + ) + + def test_doc_contains_required_data(self): + srv_types = dict() + for srv in self.data.all_services: + srv_types[srv["service_type"]] = srv + for doc in self.data.all_docs: + for attr in [ + "rst_location", + "service_type", + "title", + "type", + ]: + self.assertIn(attr, doc, f"Document {doc} contains {attr}") + self.assertIn( + doc["service_type"], + srv_types, + f"Document {doc} contains valid service_type", + ) diff --git a/releasenotes/notes/.placeholder b/releasenotes/notes/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/_static/.placeholder b/releasenotes/source/_static/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/_templates/.placeholder b/releasenotes/source/_templates/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 0000000..ecd0d77 --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'otcdocstheme', + 'reno.sphinxext', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'otc-metadata Release Notes' +copyright = '2022, Open Telekom Cloud Developers' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +# The full version, including alpha/beta/rc tags. +release = '' +# The short X.Y version. +version = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'otc-metadataReleaseNotesdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'otc-metadataReleaseNotes.tex', + 'otc-metadata Release Notes Documentation', + 'Open Telekom Cloud', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'otc-metadatarereleasenotes', + 'otc-metadata Release Notes Documentation', + ['Open Telekom Cloud'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'otc-metadata ReleaseNotes', + 'otc-metadata Release Notes Documentation', + 'Open Telekom Cloud Foundation', 'otc-metadataReleaseNotes', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 0000000..a040fa0 --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,8 @@ +============================================ + otc-metadata Release Notes +============================================ + +.. toctree:: + :maxdepth: 1 + + unreleased diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst new file mode 100644 index 0000000..cd22aab --- /dev/null +++ b/releasenotes/source/unreleased.rst @@ -0,0 +1,5 @@ +============================== + Current Series Release Notes +============================== + +.. release-notes:: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1d18dd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +pbr>=2.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ad735b1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,25 @@ +[metadata] +name = otc-metadata +summary = Metadata about OTC for Ecosystem +description_file = + README.rst +author = Open Telekom Cloud +home_page = https://open.telekom.cloud/ +python_requires = >=3.6 +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: Implementation :: CPython + +[files] +packages = + otc_metadata diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7f1bbe8 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..8a2bd6e --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,7 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +stestr>=2.0.0 # Apache-2.0 +testtools>=2.2.0 # MIT +flake8 diff --git a/tools-requirements.txt b/tools-requirements.txt new file mode 100644 index 0000000..5fee53b --- /dev/null +++ b/tools-requirements.txt @@ -0,0 +1,6 @@ +GitPython +ruamel.yaml +requests +jinja2 +dirsync +cookiecutter diff --git a/tools/bootstrap_repositories.py b/tools/bootstrap_repositories.py new file mode 100644 index 0000000..719ad38 --- /dev/null +++ b/tools/bootstrap_repositories.py @@ -0,0 +1,156 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +# import os +import pathlib +import subprocess +# import warnings + +# from git import exc +from git import Repo +# from git import SymbolicReference + +# from jinja2 import PackageLoader +# from jinja2 import Environment +# from jinja2 import select_autoescape + +from cookiecutter.main import cookiecutter + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def create_repo(repo, repo_dir, service): + repo_owner, repo_name = repo["repo"].split("/") + git_repo = None + + if repo_dir.exists(): + logging.debug(f"Repository {repo['repo']} is already existing") + return + + logging.info(f"Creating repository {repo_owner}/{repo_name}") + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + git_fqdn = "github.com" + + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.debug("Error") + pass + + if git_repo: + return + + if repo["type"] == "gitea": + subprocess.run( + args=[ + "tea", + "repo", + "create", + "--name", + repo_name, + "--owner", + repo_owner, + ] + ) + elif repo["type"] == "github": + subprocess.run( + args=[ + "gh", + "repo", + "create", + repo["repo"], + "--public", + ] + ) + cookiecutter( + template="https://github.com/opentelekomcloud/docs-cookiecutter", + output_dir=repo_dir.parent, + no_input=True, + extra_context=dict( + git_fqdn=git_fqdn, + repo_group=repo_owner, + repo_name=repo_name, + project_short_description=( + f"Documentation project for {service['service_title']} " + "service" + ), + overwrite_if_exists=True, + ), + ) + git_repo = Repo(repo_dir) + git_repo.create_remote("origin", repo_url) + git_repo.remotes.origin.fetch() + git_repo.git.push("--set-upstream", "origin", "main") + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] != args.target_environment: + continue + + checkout_exists = repo_dir.exists() + logging.debug(f"Repository {repo} exists {checkout_exists}") + repo_dir.parent.mkdir(parents=True, exist_ok=True) + if True: # not checkout_exists: + create_repo(repo, repo_dir, service) + + +def main(): + parser = argparse.ArgumentParser(description="Bootstrap repositories.") + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/convert_data.py b/tools/convert_data.py new file mode 100644 index 0000000..1d7fb3e --- /dev/null +++ b/tools/convert_data.py @@ -0,0 +1,22 @@ +import re + +import otc_metadata.services +from ruamel.yaml import YAML + +data = otc_metadata.services.Services() +new_data = data._service_data + +# services = data.service_dict + +for doc in new_data["documents"]: + hc_location = None + link = doc.get("link") + if link: + print(f"Parsing {link}") + # (p1, p2) = link.split("/") + doc["link"] = re.sub(r"/(.*)/(.*)/", r"/\2/\1/", link) + +_yaml = YAML() +_yaml.indent(mapping=2, sequence=4, offset=2) +with open("new.yaml", "w") as fd: + _yaml.dump(new_data, fd) diff --git a/tools/generate_doc_confpy.py b/tools/generate_doc_confpy.py new file mode 100644 index 0000000..7c3a5c9 --- /dev/null +++ b/tools/generate_doc_confpy.py @@ -0,0 +1,363 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import pathlib +import requests +import subprocess + +from git import exc +from git import Repo + +from jinja2 import PackageLoader +from jinja2 import Environment +from jinja2 import select_autoescape + +import otc_metadata.services + +data = otc_metadata.services.Services() + +api_session = requests.Session() + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_to = None + repo_to = None + url_to = None + target_repo = None + git_fqdn = None + + env = Environment( + loader=PackageLoader("otc_metadata"), autoescape=select_autoescape() + ) + conf_py_template = env.get_template("conf.py.j2") + tox_ini_template = env.get_template("tox.ini.j2") + index_sbv_template = env.get_template("index_sbv.rst.j2") + doc_requirements_template = env.get_template("doc_requirements.txt.j2") + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] == args.target_environment: + copy_to = repo_dir + else: + logging.debug(f"Skipping repository {repo}") + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + repo_dir.rmdir() + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.error(f"Error cloning repository {repo_url}") + return + + if repo["environment"] == args.target_environment: + url_to = repo_url + repo_to = git_repo + target_repo = repo + break + + if not target_repo: + logging.info( + f"No repository service {service['service_title']}" + f"for environment {args.target_environment}" + ) + return + + branch_name = f"{args.branch_name}" + if args.branch_force: + logging.debug("Dropping current branch") + try: + repo_to.delete_head(branch_name, force=True) + repo_to.delete_head(f"refs/heads/{args.branch_name}", force=True) + except exc.GitCommandError as e: + print(e) + pass + try: + new_branch = repo_to.create_head(branch_name, "main") + except Exception as e: + logging.warning(f"Skipping service {service} due to {e}") + return + new_branch.checkout() + + service_docs = list(data.docs_by_service_type(service["service_type"])) + + for doc in service_docs: + logging.debug(f"Analyzing document {doc}") + + conf_py_path = pathlib.Path(copy_to, doc["rst_location"], "conf.py") + if not conf_py_path.exists(): + logging.info(f"Path for document {doc['title']} does not exist") + conf_py_path.parent.mkdir(parents=True, exist_ok=True) + context = dict( + repo_name=target_repo["repo"], + project=service["service_title"], + # pdf_name=doc["pdf_name"], + title=f"{service['service_title']} - {doc['title']}", + ) + if "pdf_name" in doc: + context["pdf_name"] = doc["pdf_name"] + if git_fqdn: + context["git_fqdn"] = git_fqdn + if target_repo.get("type") != "github": + context["git_type"] = target_repo["type"] + if args.target_environment == "internal": + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-int.otc-service.com", + ) + conf_py_content = conf_py_template.render(**context) + with open(conf_py_path, "w", encoding="utf-8", newline="") as out: + logging.debug(f"Generating {conf_py_path} from template...") + out.write(conf_py_content) + + repo_to.index.add([doc["rst_location"]]) + + if args.update_sbv: + """Add or update service-based-view""" + copy_path = pathlib.Path(copy_to, 'doc', 'source') + context = dict( + repo_name=target_repo["repo"], + project=service["service_title"], + # pdf_name=doc["pdf_name"], + title=f"{service['service_title']} - Service Based View", + service_type=service["service_type"] + ) + if not copy_path.exists(): + logging.info("Path for sbv does not exist") + copy_path.mkdir(parents=True, exist_ok=True) + context["otc_sbv"] = True + if git_fqdn: + context["git_fqdn"] = git_fqdn + if target_repo.get("type") != "github": + context["git_type"] = target_repo["type"] + if args.target_environment == "internal": + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-int.otc-service.com", + ) + sbv_title = (service["service_title"] + "\n" + + ('=' * len(service["service_title"]))) + context["sbv_title"] = sbv_title + conf_py_content = conf_py_template.render(**context) + index_sbv_content = index_sbv_template.render(**context) + with open( + pathlib.Path(copy_path, "conf.py"), + "w", + encoding="utf-8") as out: + out.write(conf_py_content) + repo_to.index.add(pathlib.Path(copy_path, "conf.py")) + + if (not args.overwrite_index_sbv + and pathlib.Path(copy_path, "index.rst").exists()): + logging.info("File index.rst for sbv exists. Skipping") + else: + with open( + pathlib.Path(copy_path, "index.rst"), + "w", + encoding="utf-8") as out: + out.write(index_sbv_content) + repo_to.index.add(pathlib.Path(copy_path, "index.rst")) + + placeholder_path = pathlib.Path(copy_path, "_static") + if not pathlib.Path(placeholder_path, "placeholder").exists(): + placeholder_path.mkdir(parents=True, exist_ok=True) + open(pathlib.Path(placeholder_path, "placeholder"), 'a').close() + repo_to.index.add(pathlib.Path(placeholder_path, "placeholder")) + + if args.update_tox: + """Update tox.ini""" + context = dict(docs=[]) + for doc in service_docs: + if doc["type"] == "dev": + doc["type"] = "dev-guide" + context["docs"].append(doc) + + tox_ini_content = tox_ini_template.render(**context) + tox_ini_path = pathlib.Path(copy_to, "tox.ini") + doc_requirements_content = doc_requirements_template.render(**context) + doc_requirements_path = pathlib.Path( + copy_to, "doc", "requirements.txt" + ) + doc_requirements_path.parent.mkdir(parents=True, exist_ok=True) + with open(tox_ini_path, "w", encoding="utf-8", newline="") as out: + logging.debug(f"Generating {tox_ini_path} from template...") + out.write(tox_ini_content) + repo_to.index.add(["tox.ini"]) + with open( + doc_requirements_path, "w", encoding="utf-8", newline="" + ) as out: + logging.debug( + f"Generating {doc_requirements_path} from template..." + ) + out.write(doc_requirements_content) + repo_to.index.add(["doc/requirements.txt"]) + + if len(repo_to.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % service["service_type"] + ) + return + repo_to.index.commit( + args.commit_description + ) + push_args = ["--set-upstream", "origin", branch_name] + if args.branch_force: + push_args.append("--force") + repo_to.git.push(*push_args) + if "github" in url_to: + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=False + ) + elif "gitea" in url_to and args.token: + open_pr( + args, + repo["repo"], + dict( + title="Update Docs configuration", + head=branch_name, + ), + ) + + +def open_pr(args, repository, pr_data): + req = dict( + base=pr_data.get("base", "main"), + head=pr_data["head"], + ) + if "title" in pr_data: + req["title"] = pr_data["title"] + if "body" in pr_data: + req["body"] = pr_data["body"].replace("\\n", "\n") + if "assignees" in pr_data: + req["assignees"] = pr_data["assignees"] + if "labels" in pr_data: + req["labels"] = pr_data["labels"] + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/pulls", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + # print(f"Going to open PR with title {pr_data['title']} in {repository}") + + +def main(): + parser = argparse.ArgumentParser( + description="Update conf.py file in repositories." + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--update-tox", action="store_true", help="Whether to update tox.ini." + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="confpy", + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--branch-force", + action="store_true", + help="Whether to force branch recreation.", + ) + parser.add_argument("--token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + parser.add_argument( + "--update-sbv", + action="store_true", + help="Whether to update service-based-view" + ) + parser.add_argument( + "--overwrite-index-sbv", + action="store_true", + help=("Whether to overwrite index.rst for service-based-view." + + "\nCan only be used if --update-sbv is also specified") + ) + parser.add_argument( + "--commit-description", + default=( + "Update tox.ini && conf.py file\n\n" + "Performed-by: gitea/infra/otc-metadata-swiss/" + "tools/generate_doc_confpy.py" + ), + help="Commit description for the commit", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.overwrite_index_sbv and not args.update_sbv: + logging.error( + "Cannot overwrite index.rst for service-based-view" + + " without updating service-based-view" + ) + exit(1) + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + if args.token: + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_doc_gitcontrol_repos.py b/tools/generate_doc_gitcontrol_repos.py new file mode 100644 index 0000000..bcb63a7 --- /dev/null +++ b/tools/generate_doc_gitcontrol_repos.py @@ -0,0 +1,184 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import copy +import logging +import yaml + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_services(args, services): + """Process services""" + gitea_bp = dict( + branch_name="main", + enable_approvals_whitelist=True, + approvals_whitelist_teams=["docs-infra-team"], + block_on_rejected_reviews=True, + dismiss_stale_approvals=True, + enable_push=False, + status_check_contexts=["gl/check"], + enable_merge_whitelist=True, + merge_whitelist_usernames=["zuul"], + ) + gitea_repo_template = dict( + default_branch="main", + description="Open Telekom Cloud Service docs", + homepage=None, + archived=False, + has_issues=True, + has_projects=False, + has_wiki=False, + default_delete_branch_after_merge=True, + allow_merge_commit=False, + allow_squash_merge=True, + allow_rebase_merge=False, + default_merge_style="squash", + branch_protections=[], + ) + github_bp = dict( + branch_name="main", + enable_approvals_whitelist=True, + approvals_whitelist_teams=["docs-infra-team"], + block_on_rejected_reviews=True, + dismiss_stale_approvals=True, + enable_push=False, + status_check_contexts=["gl/check"], + enable_merge_whitelist=True, + merge_whitelist_usernames=["zuul"], + ) + github_repo_template = dict( + default_branch="main", + description="Open Telekom Cloud Service docs", + homepage=None, + archived=False, + has_issues=True, + has_projects=False, + has_wiki=False, + delete_branch_on_merge=True, + allow_merge_commit=False, + allow_squash_merge=True, + allow_rebase_merge=False, + allow_update_branch=True, + branch_protections=[], + ) + + for service in services: + logging.debug(f"Processing service {service}") + config = None + repo_name = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + if repo["environment"] != args.target_environment: + continue + + if repo["type"] == "gitea": + teams = [] + branch_protections_main = copy.deepcopy(gitea_bp) + if "teams" in repo: + teams_def = repo["teams"] + if "teams" in service: + teams_def = service["teams"] + if teams_def: + for team in teams_def: + branch_protections_main[ + "approvals_whitelist_teams" + ].append(team["name"]) + teams.append(team["name"]) + data = copy.deepcopy(gitea_repo_template) + data["description"] = ( + f"Open Telekom Cloud {service['service_title']} " + f"Service docs" + ) + data["branch_protections"].append(branch_protections_main) + data["teams"] = teams + repo_name = repo["repo"].split("/")[1] + config = {repo_name: data} + + elif repo["type"] == "github": + teams = [] + branch_protections_main = copy.deepcopy(github_bp) + if "teams" in repo: + teams_def = repo["teams"] + if "teams" in service: + teams_def = service["teams"] + if teams_def: + for team in teams_def: + teams.append( + { + "slug": team["name"], + "permission": "push" + if team["permission"] == "write" + else "pull", + } + ) + data = copy.deepcopy(github_repo_template) + data["description"] = ( + f"Open Telekom Cloud {service['service_title']} " f"docs" + ) + data["branch_protections"].append( + {"branch": "main", "template": "zuul"} + ) + data["teams"] = teams + repo_name = repo["repo"].split("/")[1] + config = {repo_name: data} + + else: + logging.error( + "Repository type %s is not supported", repo["type"] + ) + exit(1) + + if data: + with open( + f"{args.work_dir}/{repo_name}.yml", + "w", + encoding="utf-8", + newline="", + ) as out: + logging.debug( + "Generating %s config from template...", + service["service_type"], + ) + yaml.dump(config, out) + + +def main(): + parser = argparse.ArgumentParser( + description="Update conf.py file in repositories." + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + + process_services(args, data.all_services) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_docexports_data.py b/tools/generate_docexports_data.py new file mode 100644 index 0000000..8b8012a --- /dev/null +++ b/tools/generate_docexports_data.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys + +from ruamel.yaml import YAML + +import otc_metadata.services + + +def main(): + data = otc_metadata.services.Services() + + _yaml = YAML() + _yaml.indent(mapping=2, sequence=4, offset=2) + sys.stdout.write( + "# Auto-generated by otc_metadata.generate_docexports.data\n" + ) + _yaml.dump(data.docs_html_by_category("internal"), sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/tools/open_doc_issue.py b/tools/open_doc_issue.py new file mode 100644 index 0000000..bf4b34b --- /dev/null +++ b/tools/open_doc_issue.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import argparse +# import re +import requests +# import sys + +import otc_metadata.services + +api_session = requests.Session() + + +def open_issue(args, repository, issue_data): + req = dict( + title=issue_data["title"], body=issue_data["body"].replace("\\n", "\n") + ) + if "assignees" in issue_data: + req["assignees"] = issue_data["assignees"] + if "labels" in issue_data: + req["labels"] = issue_data["labels"] + print(req) + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/issues", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + print( + f"Going to open issue with title {issue_data['title']} in {repository}" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Open Issue for every document." + ) + parser.add_argument("token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + parser.add_argument("--environment", help="Environment for the repository") + parser.add_argument("--title", required=True, help="Issue title") + parser.add_argument("--body", required=True, help="Issue body") + parser.add_argument( + "--repo", + help="Repository to report issue in (instead of doc repository).", + ) + parser.add_argument( + "--assignee", + help="Issue assignee to use instead of document service assignees.", + ) + parser.add_argument( + "--labels", + help="Issue labels to use (comma separated list of label IDs).", + ) + args = parser.parse_args() + data = otc_metadata.services.Services() + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for doc in data.all_docs_full(environment=args.environment): + issue_data = dict( + title=args.title.format(**doc), + body=args.body.format(**doc), + repository=doc["repository"], + ) + if "assignees" in doc: + issue_data["assignees"] = doc["assignees"] + if args.assignee: + issue_data["assignees"] = [args.assignee] + if args.labels: + issue_data["labels"] = [int(x) for x in args.labels.split(",")] + open_issue(args, args.repo or doc["repository"], issue_data) + + +if __name__ == "__main__": + main() diff --git a/tools/service_list b/tools/service_list new file mode 100644 index 0000000..b17a6f0 --- /dev/null +++ b/tools/service_list @@ -0,0 +1,50 @@ +antiddos +aom +as +bms +cbr +cce +cdn +ces +csbs +css +cts +das +dc +dcs +dds +deh +dis +dms +dns +drs +dws +ecs +eip +elb +evs +gaussdb_mysql +gaussdb_nosql +iam +ims +kms +lts +ma +mrs +natgw +obs +opengauss +plas +rds +rts +sdrs +sfs +smn +swr +tms +vbs +vpc +vpcep +vpn +waf +wafd diff --git a/tools/sort_metadata.py b/tools/sort_metadata.py new file mode 100644 index 0000000..caeae59 --- /dev/null +++ b/tools/sort_metadata.py @@ -0,0 +1,9 @@ +# import copy + +import otc_metadata.services +# from ruamel.yaml import YAML + +data = otc_metadata.services.Services() + +data._sort_data() +data._rewrite_data() diff --git a/tools/sync_doc_repo.py b/tools/sync_doc_repo.py new file mode 100644 index 0000000..5bdbc7b --- /dev/null +++ b/tools/sync_doc_repo.py @@ -0,0 +1,201 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +import os +import pathlib +import subprocess +import warnings +from dirsync import sync + +from git import Repo +from git import SymbolicReference + + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_repositories(args, service): + """Checkout repositories""" + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_from = None + copy_to = None + repo_to = None + url_from = None + url_to = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] == args.source_environment: + copy_from = repo_dir + elif repo["environment"] == args.target_environment: + copy_to = repo_dir + else: + continue + + checkout_exists = repo_dir.exists() + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + if not checkout_exists: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + else: + logging.debug("Checkout already exists") + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + + if repo["environment"] == args.source_environment: + url_from = repo_url + elif repo["environment"] == args.target_environment: + url_to = repo_url + repo_to = git_repo + logging.debug(f"Synchronizing from={url_from}, to={url_to}") + + if not copy_from or not copy_to: + logging.warn( + "Not synchronizing documents of the service. " + "Target or source not known." + ) + + for doc in data.docs_by_service_type(service["service_type"]): + logging.debug(f"Analyzing document {doc}") + if args.document_type and doc.get("type") != args.document_type: + logging.info( + f"Skipping synchronizing {doc['title']} " + f"due to the doc-type filter." + ) + continue + if doc.get("environment"): + logging.info( + f"Skipping synchronizing {doc['title']} " + f"since it is environment bound." + ) + continue + + branch_name = f"{args.branch_name}#{doc['type']}" + remote_ref = SymbolicReference.create( + repo_to, "refs/remotes/origin/%s" % branch_name + ) + new_branch = repo_to.create_head(branch_name, "main") + remote_ref = repo_to.remotes[0].refs[branch_name] # get correct type + new_branch.set_tracking_branch(remote_ref) + new_branch.checkout() + + source_path = pathlib.Path(copy_from, doc["rst_location"]) + target_path = pathlib.Path(copy_to, doc["rst_location"]) + sync( + source_path, + target_path, + 'sync', + purge=True, + create=True, + content=True, + ignore=['conf.py'] + ) + repo_to.index.add([doc["rst_location"]]) + + for obj in repo_to.index.diff(None).iter_change_type('D'): + repo_to.index.remove([obj.b_path]) + if len(repo_to.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug("No changes.") + continue + repo_to.index.commit( + ( + f"Synchronize {doc['title']}\n\n" + f"Overwriting document\n" + f"from: {url_from}\n" + f"to: {url_to}\n\n" + "Performed-by: gitea/infra/otc-metadata-swiss/tools/sync_doc_repo.py" + ) + ) + repo_to.remotes.origin.push(new_branch) + if "github" in url_to and args.open_pr_gh: + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Synchronize document between environments." + ) + parser.add_argument( + "--source-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--target-environment", + required=True, + help="Environment to be used as a source", + ) + parser.add_argument( + "--service-type", + required=True, + help="Service to which document(s) belongs to", + ) + parser.add_argument( + "--document-type", + help=( + "Type of the document to synchronize. " + "All will be synchronized if not set." + ), + ) + parser.add_argument( + "--branch-name", + required=True, + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--open-pr-gh", + action="store_true", + help="Open Pull Request using `gh` utility (need to be present).", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + service = data.service_dict.get(args.service_type) + if not service: + warnings.warn(f"Service {args.service_type} was not found in metadata") + os.exit(1) + + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tools/update_zuul_project_configs.py b/tools/update_zuul_project_configs.py new file mode 100644 index 0000000..8a898d9 --- /dev/null +++ b/tools/update_zuul_project_configs.py @@ -0,0 +1,294 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +# import os +import pathlib +import requests +import subprocess +# import warnings + +from git import exc +from git import Repo +# from git import SymbolicReference + +from ruamel.yaml import CommentedMap +from ruamel.yaml import YAML + +import otc_metadata.services + +data = otc_metadata.services.Services() + +yaml = YAML() + +api_session = requests.Session() + + +def load_zuul_config(workdir): + for path in [".zuul.yaml", "zuul.yaml"]: + test_path = pathlib.Path(workdir, path) + if test_path.exists(): + with open(test_path, "r") as f: + zuul_config = yaml.load(f) + return (zuul_config, test_path) + + +def open_pr(args, repository, pr_data): + req = dict( + title=pr_data["title"], + body=pr_data["body"].replace("\\n", "\n"), + base=pr_data.get("base", "main"), + head=pr_data["head"], + ) + if "assignees" in pr_data: + req["assignees"] = pr_data["assignees"] + if "labels" in pr_data: + req["labels"] = pr_data["labels"] + rsp = api_session.post( + f"{args.api_url}/repos/{repository}/pulls", json=req + ) + if rsp.status_code != 201: + print(rsp.text) + print(f"Going to open PR with title {pr_data['title']} in {repository}") + + +def process_repositories(args, service): + """Checkout repositories""" + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_to = None + # repo_to = None + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] != args.environment: + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.update() + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + repo_dir.rmdir() + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception: + logging.error(f"Error cloning repository {repo_url}") + return + + branch_name = f"{args.branch_name}" + if args.branch_force: + logging.debug("Dropping current branch") + try: + git_repo.delete_head(branch_name, force=True) + except exc.GitCommandError: + pass + try: + new_branch = git_repo.create_head(branch_name, "main") + except Exception as ex: + logging.warning(f"Skipping service {service} due to {ex}") + return + new_branch.checkout() + + (zuul_config, zuul_file_name) = load_zuul_config(repo_dir) + zuul_templates = None + zuul_jobs = dict() + zuul_new_jobs = list() + zuul_config_updated = False + for item in zuul_config: + if "project" in item.keys(): + project = item["project"] + zuul_templates = project.setdefault("templates", []) + if not zuul_templates: + zuul_templates = [] + elif "job" in item.keys(): + job = item["job"] + zuul_jobs[job["name"]] = job + print(f"Existing jobs {zuul_jobs}") + if "helpcenter-base-jobs" not in zuul_templates: + zuul_templates.append("helpcenter-base-jobs") + zuul_config_updated = True + + job_suffix = ( + "-hc-int-jobs" if args.environment == "internal" else "-hc-jobs" + ) + for doc in data.docs_by_service_type(service["service_type"]): + logging.debug(f"Analyzing document {doc}") + if not doc.get("type"): + continue + if doc["type"] == "dev": + doc_type = "dev-guide" + else: + doc_type = doc["type"] + template_name = f"{doc_type}{job_suffix}" + if doc_type in ["api-ref", "umn", "dev-guide"]: + if template_name not in zuul_templates: + zuul_templates.append(template_name) + else: + job_name = f"build-otc-{doc['service_type']}-{doc_type}" + if job_name not in zuul_jobs: + zuul_config_updated = True + zuul_new_jobs.append( + dict( + job=dict( + name=job_name, + parent="otc-tox-docs", + description=( + f"Build {doc_type} document using tox" + ), + files=[ + f"^{doc['rst_location']}/.*" + ], + vars=dict( + tox_envlist=doc_type + ) + ) + ) + ) + + if zuul_config_updated: + for new_job in zuul_new_jobs: + zuul_config.insert(0, new_job) + + for item in zuul_config: + if "project" in item.keys(): + project = item["project"] + project["templates"] = zuul_templates + # Ensure new jobs are in check + if len(zuul_new_jobs) > 0: + project.setdefault( + "check", + CommentedMap(jobs=[]) + ) + project["check"].yaml_set_comment_before_after_key( + key="jobs", + indent=6, + before=( + "Separate documents are rendered in check, " + "while published through regular " + "otc-tox-docs job part of the basic template" + ) + ) + project["check"]["jobs"].extend( + [x["job"]["name"] for x in zuul_new_jobs]) + + # yaml.indent(offset=2, sequence=2) + with open(zuul_file_name, "w") as f: + yaml.dump(zuul_config, f) + git_repo.index.add([zuul_file_name.name]) + if len(git_repo.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % + service["service_type"]) + return + + git_repo.index.commit( + ( + "Update zuul.yaml file\n\n" + "Performed-by: gitea/infra/otc-metadata-swiss" + "/tools/update_zuul_project_config.py" + ) + ) + push_args = ["--set-upstream", "origin", branch_name] + if args.branch_force: + push_args.append("--force") + + git_repo.git.push(*push_args) + if repo["type"] == "github": + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True + ) + elif repo["type"] == "gitea": + open_pr( + args, + repo["repo"], + dict( + title="Update zuul config", + body="Update zuul config templates", + head=branch_name, + ), + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Update zuul.yaml file in repositories." + ) + parser.add_argument( + "--environment", + required=True, + help="Repository Environment", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="zuul", + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--branch-force", + action="store_true", + help="Whether to force branch recreation.", + ) + parser.add_argument("--token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + if args.token: + api_session.headers.update({"Authorization": f"token {args.token}"}) + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..3b5961d --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +minversion = 3.2.0 +envlist = py3,pep8 +ignore_basepython_conflict = true + +[testenv] +basepython = python3 +usedevelop = True +setenv = + PYTHONWARNINGS=default::DeprecationWarning +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run {posargs} + +[testenv:pep8] +commands = flake8 {posargs} + +[testenv:venv] +commands = {posargs} + +[flake8] +# E123, E125 skipped as they are invalid PEP-8. +show-source = True +ignore = E123,E125,W503 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build diff --git a/zuul.yaml b/zuul.yaml new file mode 100644 index 0000000..3a1ad47 --- /dev/null +++ b/zuul.yaml @@ -0,0 +1,12 @@ +--- +- project: + merge-mode: squash-merge + default-branch: main + check: + jobs: + - otc-tox-pep8 + - otc-tox-py39 + gate: + jobs: + - otc-tox-pep8 + - otc-tox-py39