From a092056387c74b8d416c319cb51d2877b5dd55b1 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Thu, 1 Sep 2022 09:04:48 +0200 Subject: [PATCH] add new tools --- otc_metadata/data/services.yaml | 16 +-- otc_metadata/templates/conf.py.j2 | 103 +++++++++++++++ tools/bootstrap_repositories.py | 156 ++++++++++++++++++++++ tools/generate_doc_confpy.py | 207 ++++++++++++++++++++++++++++++ tools/sync_doc_repo.py | 2 +- 5 files changed, 472 insertions(+), 12 deletions(-) create mode 100644 otc_metadata/templates/conf.py.j2 create mode 100644 tools/bootstrap_repositories.py create mode 100644 tools/generate_doc_confpy.py diff --git a/otc_metadata/data/services.yaml b/otc_metadata/data/services.yaml index ebee4bcb..ce9a6147 100644 --- a/otc_metadata/data/services.yaml +++ b/otc_metadata/data/services.yaml @@ -89,14 +89,14 @@ documents: title: User Guide type: umn - html_location: docs/cce/api-ref - link: /api-ref/cloud-container-service/ + link: /api-ref/cloud-container-engine/ pdf_name: cce-api-ref rst_location: api-ref/source service_type: cce title: API Reference type: api-ref - html_location: docs/cce/umn - link: /umn/cloud-container-service/ + link: /umn/cloud-container-engine/ pdf_name: cce-umn rst_location: umn/source service_type: cce @@ -324,12 +324,6 @@ documents: service_type: ecs title: Developer Guide type: dev - - environment: internal - link: /not_present/elastic-cloud-server/ - rst_location: docs/ecs/hld - service_type: ecs - title: HLD - type: hld - html_location: docs/ecs/umn link: /umn/elastic-cloud-server/ pdf_name: ecs-umn @@ -855,10 +849,10 @@ services: service_type: cbr - repositories: - environment: internal - repo: docs/cloud-container-service + repo: docs/cloud-container-engine type: gitea - environment: public - repo: opentelekomcloud-docs/cloud-container-service + repo: opentelekomcloud-docs/cloud-container-engine type: github service_category: container service_title: Cloud Container Service @@ -1274,7 +1268,7 @@ services: - environment: internal repo: docs/volume-backup-service type: gitea - - env: pub + - environment: public repo: opentelekomcloud-docs/volume-backup-service type: github service_category: storage diff --git a/otc_metadata/templates/conf.py.j2 b/otc_metadata/templates/conf.py.j2 new file mode 100644 index 00000000..8aecb78d --- /dev/null +++ b/otc_metadata/templates/conf.py.j2 @@ -0,0 +1,103 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# !!! +# This file is generated out of template in doc-exports repository. +# Beware overwriting it locally. + +import os +import sys + +extensions = [ + 'otcdocstheme' +] + +otcdocs_auto_name = False +otcdocs_auto_version = False + +project = '{{ project }}' +otcdocs_repo_name = '{{ repo_name }}' +# Those variables are required for edit/bug links +{% if git_fqdn -%} +otcdocs_git_fqdn = '{{ git_fqdn }}' +{% endif %} +{% if git_type -%} +otcdocs_git_type = '{{ git_type }}' +{% endif %} + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) + +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +copyright = u'2022-present, Open Telekom Cloud' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +language = 'en' + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { +} + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +{% if title is defined %} +html_title = "{{ title }}" +{% endif %} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# -- Options for PDF output -------------------------------------------------- +latex_documents = [ +{% if pdf_name is defined -%} + ('index', + '{{ pdf_name }}.tex', + u'{{ title }}', + u'OpenTelekomCloud', 'manual'), +{% endif -%} +] diff --git a/tools/bootstrap_repositories.py b/tools/bootstrap_repositories.py new file mode 100644 index 00000000..85f6d082 --- /dev/null +++ b/tools/bootstrap_repositories.py @@ -0,0 +1,156 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import pathlib +import subprocess +import warnings + +from git import exc +from git import Repo +from git import SymbolicReference + +from jinja2 import PackageLoader +from jinja2 import Environment +from jinja2 import select_autoescape + +from cookiecutter.main import cookiecutter + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def create_repo(repo, repo_dir, service): + repo_owner, repo_name = repo['repo'].split('/') + git_repo = None + + if repo_dir.exists(): + logging.debug(f"Repository {repo['repo']} is already existing") + return + + logging.info(f"Creating repository {repo_owner}/{repo_name}") + if repo['type'] == 'gitea': + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo['type'] == 'github': + repo_url = f"git@github.com:/{repo['repo']}" + git_fqdn = "github.com" + + try: + git_repo = Repo.clone_from( + repo_url, + repo_dir, branch='main') + except Exception: + logging.debug("Error") + pass + + if git_repo: + return + + if repo["type"] == "gitea": + subprocess.run( + args=["tea", "repo", "create", + "--name", repo_name, + "--owner", repo_owner, + ] + ) + elif repo["type"] == "github": + subprocess.run( + args=["gh", "repo", "create", + repo["repo"], "--public", + ] + ) + cookiecutter( + template="https://github.com/opentelekomcloud/docs-cookiecutter", + output_dir=repo_dir.parent, + no_input=True, + extra_context=dict( + git_fqdn=git_fqdn, + repo_group=repo_owner, + repo_name=repo_name, + project_short_description=( + f"Documentation project for {service['service_title']} " + "service"), + overwrite_if_exists=True + ) + ) + git_repo = Repo(repo_dir) + git_repo.create_remote('origin', repo_url) + git_repo.remotes.origin.fetch() + git_repo.git.push('--set-upstream', 'origin', 'main') + + +def process_repositories(args, service): + """Checkout repositories + + """ + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path( + workdir, repo["type"], repo["repo"] + ) + + if repo["environment"] != args.target_environment: + continue + + checkout_exists = repo_dir.exists() + logging.debug(f"Repository {repo} exists {checkout_exists}") + repo_dir.parent.mkdir(parents=True, exist_ok=True) + if True: # not checkout_exists: + create_repo(repo, repo_dir, service) + + +def main(): + parser = argparse.ArgumentParser( + description='Bootstrap repositories.') + parser.add_argument( + '--target-environment', + required=True, + help='Environment to be used as a source' + ) + parser.add_argument( + '--service-type', + help='Service to update' + ) + parser.add_argument( + '--work-dir', + required=True, + help='Working directory to use for repository checkout.' + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + for service in services: + process_repositories(args, service) + + +if __name__ == '__main__': + main() diff --git a/tools/generate_doc_confpy.py b/tools/generate_doc_confpy.py new file mode 100644 index 00000000..6b6ba98b --- /dev/null +++ b/tools/generate_doc_confpy.py @@ -0,0 +1,207 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import pathlib +import subprocess +import warnings + +from git import exc +from git import Repo +from git import SymbolicReference + +from jinja2 import PackageLoader +from jinja2 import Environment +from jinja2 import select_autoescape + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_repositories(args, service): + """Checkout repositories + + """ + logging.debug(f"Processing service {service}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + copy_to = None + repo_to = None + url_to = None + target_repo = None + git_fqdn = None + + env = Environment( + loader=PackageLoader("otc_metadata"), + autoescape=select_autoescape()) + conf_py_template = env.get_template("conf.py.j2") + + for repo in service["repositories"]: + logging.debug(f"Processing repository {repo}") + repo_dir = pathlib.Path( + workdir, repo["type"], repo["repo"] + ) + + if repo["environment"] == args.target_environment: + copy_to = repo_dir + else: + logging.debug(f"Skipping repository {repo}") + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == 'gitea': + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + git_fqdn = "gitea.eco.tsi-dev.otc-service.com" + elif repo["type"] == 'github': + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + exit(1) + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + repo_dir.rmdir() + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from( + repo_url, + repo_dir, branch='main') + except Exception: + logging.error(f"Error cloning repository {repo_url}") + return + + if repo["environment"] == args.target_environment: + url_to = repo_url + repo_to = git_repo + target_repo = repo + break + + if not target_repo: + logging.info( + f"No repository service {service['service_title']}" + f"for environment {args.target_environment}") + return + + branch_name = f"{args.branch_name}" + if args.branch_force: + logging.debug("Dropping current branch") + try: + repo_to.delete_head(branch_name) + except exc.GitCommandError: + pass + try: + new_branch = repo_to.create_head(branch_name, 'main') + except Exception: + logging.warn(f"Skipping service {service}") + return + new_branch.checkout() + + for doc in data.docs_by_service_type(service["service_type"]): + logging.debug(f"Analyzing document {doc}") + + conf_py_path = pathlib.Path(copy_to, doc['rst_location'], 'conf.py') + if not conf_py_path.exists(): + logging.info(f"Path for document {doc['title']} does not exist") + conf_py_path.parent.mkdir(parents=True, exist_ok=True) + context = dict( + repo_name=target_repo["repo"], + project=service["service_title"], + pdf_name=doc["pdf_name"], + title=f"{service['service_title']} - {doc['title']}" + ) + if git_fqdn: + context["git_fqdn"] = git_fqdn + if target_repo.get("type") != "github": + context["git_type"] = target_repo["type"] + conf_py_content = conf_py_template.render(**context) + with open(conf_py_path, 'w', encoding='utf-8', newline='') as out: + logging.debug(f"Generating {conf_py_path} from template...") + out.write(conf_py_content) + + repo_to.index.add([doc["rst_location"]]) + repo_to.index.commit( + ( + f"Update conf.py file\n\n" + "Performed-by: gitea/infra/otc-metadata/tools/generate_doc_confpy.py" + ) + ) + push_args = ("--set-upstream", "origin", branch_name) + repo_to.git.push(*push_args) + if 'github' in url_to: + subprocess.run( + args=['gh', 'pr', 'create', '-f'], + cwd=copy_to, + check=True + ) + + +def main(): + parser = argparse.ArgumentParser( + description='Update conf.py file in repositories.') + parser.add_argument( + '--target-environment', + required=True, + help='Environment to be used as a source' + ) + parser.add_argument( + '--service-type', + help='Service to update' + ) + parser.add_argument( + '--work-dir', + required=True, + help='Working directory to use for repository checkout.' + ) + parser.add_argument( + '--branch-name', + default='confpy', + help='Branch name to be used for synchronizing.' + ) + parser.add_argument( + '--branch-force', + action='store_true', + help='Whether to force branch recreation.' + ) + + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + if args.service_type: + services = [data.service_dict.get(args.service_type)] + else: + services = data.all_services + + for service in services: + process_repositories(args, service) + + +if __name__ == '__main__': + main() diff --git a/tools/sync_doc_repo.py b/tools/sync_doc_repo.py index bf45639e..6269e122 100644 --- a/tools/sync_doc_repo.py +++ b/tools/sync_doc_repo.py @@ -67,7 +67,7 @@ def process_repositories(args, service): elif repo["type"] == 'github': repo_url = f"git@github.com:/{repo['repo']}" else: - logging.error(f"Repository typw {repo['type']} is not supported") + logging.error(f"Repository type {repo['type']} is not supported") exit(1) if not checkout_exists: git_repo = Repo.clone_from(