From 1bec551697cc06e43817d148c4a6e884265f04b0 Mon Sep 17 00:00:00 2001 From: "Goncharov, Artem" Date: Thu, 28 Jul 2022 12:40:22 +0000 Subject: [PATCH] Support sync with deletion Reviewed-by: gtema Co-authored-by: Goncharov, Artem Co-committed-by: Goncharov, Artem --- docs_big-data.yaml | 10 ++- docs_compute.yaml | 22 +++-- otc_doc_convertor/convertor.py | 46 +++++++++-- playbooks/post.yaml | 2 +- playbooks/run.yaml | 17 ++-- requirements.txt | 1 + roles/convert_doc/defaults/main.yaml | 1 + roles/convert_doc/tasks/main.yaml | 24 +++++- roles/fetch_doc_artifacts/tasks/doc.yaml | 4 +- roles/fetch_doc_artifacts/tasks/patch.yaml | 4 +- templates/_static/.placeholder | 0 templates/conf.py | 95 ++++++++++++++++++++++ templates/docutils.conf | 2 + tox.ini | 2 +- 14 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 templates/_static/.placeholder create mode 100644 templates/conf.py create mode 100644 templates/docutils.conf diff --git a/docs_big-data.yaml b/docs_big-data.yaml index 4b1efad7..3fb61c9f 100644 --- a/docs_big-data.yaml +++ b/docs_big-data.yaml @@ -1,16 +1,18 @@ --- docs: - label: ma_umn - service: ma - type: umn + service_type: ma + doc_type: umn html_location: docs/modelarts/umn repository: docs/modelarts project_location: /umn/source title: ModelArts - User Guide + service: ModelArts - label: ma_api - service: ma - type: api-ref + service_type: ma + doc_type: api-ref html_location: docs/modelarts/api-ref repository: docs/modelarts project_location: /api-ref/source title: ModelArts - API Reference + service: ModelArts diff --git a/docs_compute.yaml b/docs_compute.yaml index 6f6cc2b4..1f9fe13c 100644 --- a/docs_compute.yaml +++ b/docs_compute.yaml @@ -1,30 +1,34 @@ --- docs: - label: ecs_umn - service: ecs - type: umn + service_type: ecs + doc_type: umn html_location: docs/ecs/umn repository: docs/elastic-cloud-server project_location: /umn/source - title: Elastic Cloud Server - User Manual + title: Elastic Cloud Server - User Guide + service: Elastic Cloud Server - label: ecs_api - service: ecs - type: api-ref + service_type: ecs + doc_type: api-ref html_location: docs/ecs/api-ref repository: docs/elastic-cloud-server project_location: /api-ref/source title: Elastic Cloud Server - API Reference + service: Elastic Cloud Server - label: ecs_dev - service: ecs - type: dev-guide + service_type: ecs + doc_type: dev-guide html_location: docs/ecs/dev repository: docs/elastic-cloud-server project_location: /dev_guide/source title: Elastic Cloud Server - Developer Guide + service: Elastic Cloud Server - label: ims_umn - service: ims - type: umn + service_type: ims + doc_type: umn html_location: docs/ims/umn repository: docs/image-management-service project_location: /umn/source title: Image Management Service - User Guide + service: Image Management Service diff --git a/otc_doc_convertor/convertor.py b/otc_doc_convertor/convertor.py index b7b0ef8d..e3d8c575 100644 --- a/otc_doc_convertor/convertor.py +++ b/otc_doc_convertor/convertor.py @@ -8,7 +8,8 @@ import os import pathlib import re import shutil -import sys + +from jinja2 import FileSystemLoader, Environment, select_autoescape class OTCDocConvertor: @@ -261,8 +262,21 @@ class OTCDocConvertor: help='Directory to write resulting files') parser.add_argument( '--title', - default="Main Index", + required=True, help='Document title') + parser.add_argument( + '--service', + help='Service to which the document belongs to') + parser.add_argument( + '--repo-name', + help='Service repository') + parser.add_argument( + '--pdf-name', + help='PDF File name') + parser.add_argument( + '--templates-location', + default='templates', + help='Location of additional templates') self.args = parser.parse_args() if self.args.dest: dest = pathlib.Path(self.args.dest) @@ -272,6 +286,7 @@ class OTCDocConvertor: metadata_file = pathlib.Path( self.args.path, "CLASS.TXT.json") + meta_data = dict() if not metadata_file.exists(): logging.warning( @@ -282,8 +297,8 @@ class OTCDocConvertor: index.write(self.args.title + '\n') index.write('=' * (len(self.args.title)) + '\n') index.write('\n') - sys.exit(0) - meta_data = json.loads(open(metadata_file).read()) + else: + meta_data = json.loads(open(metadata_file).read()) metadata_by_uri = dict() metadata_by_code = dict() self.doc_images = set() @@ -385,7 +400,7 @@ class OTCDocConvertor: as reader, \ open(pathlib.Path(dest, target_path, f"{target}.rst"), 'w') as writer: - logging.info(f"Post processing {target}") + logging.info(f"Post processing {target}...") writer.write(f":original_name: {f.name}\n\n") # Add root file label writer.write(f".. _{f.name.replace('.html', '')}:\n\n") @@ -457,7 +472,7 @@ class OTCDocConvertor: # Copy used images if len(self.doc_images) > 0: - logging.debug("Processing images") + logging.debug("Processing images...") img_dest = pathlib.Path(dest, '_static', 'images') img_dest.mkdir(parents=True, exist_ok=True) for img in self.doc_images: @@ -466,6 +481,25 @@ class OTCDocConvertor: pathlib.Path(img_dest, img).resolve(strict=False) ) + context = dict( + title=self.args.title, + project=self.args.service, + repo_name=self.args.repo_name, + pdf_name=self.args.pdf_name + ) + loader = FileSystemLoader([self.args.templates_location]) + env = Environment(loader=loader, autoescape=select_autoescape()) + for f in loader.list_templates(): + outfile_tmpl = env.get_template(f) + outfile_rendered = outfile_tmpl.render(**context) + target_file = pathlib.Path(self.args.dest, f) + target_file.parent.mkdir(parents=True, exist_ok=True) + with open( + target_file, 'w', encoding='utf-8', newline='' + ) as out: + logging.debug(f"Generating {f} from template...") + out.write(outfile_rendered) + def main(): OTCDocConvertor().main() diff --git a/playbooks/post.yaml b/playbooks/post.yaml index fc9c58ef..d535a6f3 100644 --- a/playbooks/post.yaml +++ b/playbooks/post.yaml @@ -9,7 +9,7 @@ name: "fetch_doc_artifacts" vars: fetch_doc_artifacts_name: "{{ doc.label }}" - fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}" + fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ doc.repository }}/{{ doc.project_location }}" # fetch_doc_artifacts_diff_patch: "{{ ansible_user_dir }}/{{ doc.label }}.patch" # fetch_doc_artifacts_git_patch: "{{ ansible_user_dir }}/{{ doc.label }}.git.patch" loop: "{{ docs }}" diff --git a/playbooks/run.yaml b/playbooks/run.yaml index 9c7a9d2c..1742c3fc 100644 --- a/playbooks/run.yaml +++ b/playbooks/run.yaml @@ -16,12 +16,6 @@ register: git_log ignore_errors: true - - name: Configure git user name - command: "git config --global user.name 'OpenTelekomCloud Proposal Bot'" - - - name: Configure git user email - command: "git config --global user.email otcbot@otc-service.com" - - name: Generate RSTs include_role: name: "convert_doc" @@ -29,13 +23,22 @@ doc_label: "{{ doc.label }}" dest: "{{ ansible_user_dir }}/{{ doc.repository }}/{{ doc.project_location }}" source: "{{ zuul.project.src_dir }}/{{ doc.html_location }}" - title: "{{ doc.title | default(omit) }}" + doc_title: "{{ doc.title | default(omit) }}" + doc_service: "{{ doc.service | default(omit) }}" + doc_repo_name: "{{ doc.repository | default(omit) }}" + doc_pdf_name: "{{ [doc.doc_type, doc.service_type] | join('-') }}" loop: "{{ docs }}" loop_control: loop_var: "doc" label: "{{ doc.label }}" when: "git_log.stdout is search(doc.html_location) or git_log.stdout is search('otc_doc_convertor') or git_log.stdout is search('playbooks')" + # - name: Configure git user name + # command: "git config --global user.name 'OpenTelekomCloud Proposal Bot'" + # + # - name: Configure git user email + # command: "git config --global user.email otcbot@otc-service.com" + # # - name: Restore to the previous git state # command: "git checkout HEAD^1" # args: diff --git a/requirements.txt b/requirements.txt index 3b9bcf64..4db383df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ bs4 lxml +Jinja2 diff --git a/roles/convert_doc/defaults/main.yaml b/roles/convert_doc/defaults/main.yaml index 395327d0..80c25ab0 100644 --- a/roles/convert_doc/defaults/main.yaml +++ b/roles/convert_doc/defaults/main.yaml @@ -1,2 +1,3 @@ +zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" convert_params: "" diff --git a/roles/convert_doc/tasks/main.yaml b/roles/convert_doc/tasks/main.yaml index 3756ca63..1b4d0a59 100644 --- a/roles/convert_doc/tasks/main.yaml +++ b/roles/convert_doc/tasks/main.yaml @@ -8,8 +8,28 @@ - name: Add title to parameters ansible.builtin.set_fact: - convert_params: "{{ convert_params }} --title '{{ title }}'" - when: "title is defined" + convert_params: "{{ convert_params }} --title '{{ doc_title }}'" + when: "doc_title is defined" + +- name: Add service to parameters + ansible.builtin.set_fact: + convert_params: "{{ convert_params }} --service '{{ doc_service }}'" + when: "doc_service is defined" + +- name: Add repo_name to parameters + ansible.builtin.set_fact: + convert_params: "{{ convert_params }} --repo-name '{{ doc_repo_name }}'" + when: "doc_repo_name is defined" + +- name: Add pdf_name to parameters + ansible.builtin.set_fact: + convert_params: "{{ convert_params }} --pdf-name {{ doc_pdf_name }}" + when: "doc_pdf_name is defined" + +- name: Add templates-location to parameters + ansible.builtin.set_fact: + convert_params: "{{ convert_params }} --templates-location {{ zuul_work_dir }}/templates" + - name: Convert {{ doc_label | default('') }} HTML to RST args: diff --git a/roles/fetch_doc_artifacts/tasks/doc.yaml b/roles/fetch_doc_artifacts/tasks/doc.yaml index 90761ae0..cb8f1461 100644 --- a/roles/fetch_doc_artifacts/tasks/doc.yaml +++ b/roles/fetch_doc_artifacts/tasks/doc.yaml @@ -38,6 +38,6 @@ url: "{{ fetch_doc_artifacts_name }}.tar.gz" metadata: type: docs_archive - doc_service: "{{ doc.service }}" - doc_type: "{{ doc.type }}" + doc_service: "{{ doc.service_type }}" + doc_type: "{{ doc.doc_type }}" doc_label: "{{ fetch_doc_artifacts_name }}" diff --git a/roles/fetch_doc_artifacts/tasks/patch.yaml b/roles/fetch_doc_artifacts/tasks/patch.yaml index cdada8fc..431234e5 100644 --- a/roles/fetch_doc_artifacts/tasks/patch.yaml +++ b/roles/fetch_doc_artifacts/tasks/patch.yaml @@ -27,6 +27,6 @@ url: "{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch" metadata: type: "patch" - doc_service: "{{ doc.service }}" - doc_type: "{{ doc.type }}" + doc_service: "{{ doc.service_type }}" + doc_type: "{{ doc.doc_type }}" patch_type: "{{ patch_type }}" diff --git a/templates/_static/.placeholder b/templates/_static/.placeholder new file mode 100644 index 00000000..e69de29b diff --git a/templates/conf.py b/templates/conf.py new file mode 100644 index 00000000..5ab57950 --- /dev/null +++ b/templates/conf.py @@ -0,0 +1,95 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# !!! +# This file is generated out of template in doc-exports repository. +# Beware overwriting it locally. + +import os +import sys + +extensions = [ + 'otcdocstheme' +] + +otcdocs_auto_name = False +otcdocs_auto_version = False + +project = '{{ project }}' +otcdocs_repo_name = '{{ repo_name }}' + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) + +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +copyright = u'2022-present, Open Telekom Cloud' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +language = 'en' + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +html_theme = 'otcdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = {} + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +{% if title is defined %} +html_title = "{{ title }}" +{% endif %} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# -- Options for PDF output -------------------------------------------------- +latex_documents = [ +{% if pdf_name is defined -%} + ('index', + '{{ pdf_name }}.tex', + u'{{ title }}', + u'OpenTelekomCloud', 'manual'), +{% endif -%} +] diff --git a/templates/docutils.conf b/templates/docutils.conf new file mode 100644 index 00000000..808c9d27 --- /dev/null +++ b/templates/docutils.conf @@ -0,0 +1,2 @@ +[html writers] +table-style: table, caption-top diff --git a/tox.ini b/tox.ini index 07f209c7..f922100e 100644 --- a/tox.ini +++ b/tox.ini @@ -37,4 +37,4 @@ commands = {posargs} # breaks should occur before the binary operator for readability. ignore = H306,H4,W503 show-source = True -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,templates