Support sync with deletion

Reviewed-by: gtema <artem.goncharov@gmail.com>
Co-authored-by: Goncharov, Artem <artem.goncharov@t-systems.com>
Co-committed-by: Goncharov, Artem <artem.goncharov@t-systems.com>

docs_big-data.yaml

@@ -1,16 +1,18 @@
 ---
 docs:
   - label: ma_umn
-    service: ma
-    type: umn
+    service_type: ma
+    doc_type: umn
     html_location: docs/modelarts/umn
     repository: docs/modelarts
     project_location: /umn/source
     title: ModelArts - User Guide
+    service: ModelArts
   - label: ma_api
-    service: ma
-    type: api-ref
+    service_type: ma
+    doc_type: api-ref
     html_location: docs/modelarts/api-ref
     repository: docs/modelarts
     project_location: /api-ref/source
     title: ModelArts - API Reference
+    service: ModelArts

docs_compute.yaml

@@ -1,30 +1,34 @@
 ---
 docs:
   - label: ecs_umn
-    service: ecs
-    type: umn
+    service_type: ecs
+    doc_type: umn
     html_location: docs/ecs/umn
     repository: docs/elastic-cloud-server
     project_location: /umn/source
-    title: Elastic Cloud Server - User Manual
+    title: Elastic Cloud Server - User Guide
+    service: Elastic Cloud Server
   - label: ecs_api
-    service: ecs
-    type: api-ref
+    service_type: ecs
+    doc_type: api-ref
     html_location: docs/ecs/api-ref
     repository: docs/elastic-cloud-server
     project_location: /api-ref/source
     title: Elastic Cloud Server - API Reference
+    service: Elastic Cloud Server
   - label: ecs_dev
-    service: ecs
-    type: dev-guide
+    service_type: ecs
+    doc_type: dev-guide
     html_location: docs/ecs/dev
     repository: docs/elastic-cloud-server
     project_location: /dev_guide/source
     title: Elastic Cloud Server - Developer Guide
+    service: Elastic Cloud Server
   - label: ims_umn
-    service: ims
-    type: umn
+    service_type: ims
+    doc_type: umn
     html_location: docs/ims/umn
     repository: docs/image-management-service
     project_location: /umn/source
     title: Image Management Service - User Guide
+    service: Image Management Service

otc_doc_convertor/convertor.py

@@ -8,7 +8,8 @@ import os
 import pathlib
 import re
 import shutil
-import sys
+
+from jinja2 import FileSystemLoader, Environment, select_autoescape
 
 
 class OTCDocConvertor:
@@ -261,8 +262,21 @@ class OTCDocConvertor:
             help='Directory to write resulting files')
         parser.add_argument(
             '--title',
-            default="Main Index",
+            required=True,
             help='Document title')
+        parser.add_argument(
+            '--service',
+            help='Service to which the document belongs to')
+        parser.add_argument(
+            '--repo-name',
+            help='Service repository')
+        parser.add_argument(
+            '--pdf-name',
+            help='PDF File name')
+        parser.add_argument(
+            '--templates-location',
+            default='templates',
+            help='Location of additional templates')
         self.args = parser.parse_args()
         if self.args.dest:
             dest = pathlib.Path(self.args.dest)
@@ -272,6 +286,7 @@ class OTCDocConvertor:
 
         metadata_file = pathlib.Path(
             self.args.path, "CLASS.TXT.json")
+        meta_data = dict()
 
         if not metadata_file.exists():
             logging.warning(
@@ -282,8 +297,8 @@ class OTCDocConvertor:
                 index.write(self.args.title + '\n')
                 index.write('=' * (len(self.args.title)) + '\n')
                 index.write('\n')
-            sys.exit(0)
-        meta_data = json.loads(open(metadata_file).read())
+        else:
+            meta_data = json.loads(open(metadata_file).read())
         metadata_by_uri = dict()
         metadata_by_code = dict()
         self.doc_images = set()
@@ -385,7 +400,7 @@ class OTCDocConvertor:
                  as reader, \
                  open(pathlib.Path(dest, target_path,
                       f"{target}.rst"), 'w') as writer:
-                logging.info(f"Post processing {target}")
+                logging.info(f"Post processing {target}...")
                 writer.write(f":original_name: {f.name}\n\n")
                 # Add root file label
                 writer.write(f".. _{f.name.replace('.html', '')}:\n\n")
@@ -457,7 +472,7 @@ class OTCDocConvertor:
 
         # Copy used images
         if len(self.doc_images) > 0:
-            logging.debug("Processing images")
+            logging.debug("Processing images...")
             img_dest = pathlib.Path(dest, '_static', 'images')
             img_dest.mkdir(parents=True, exist_ok=True)
             for img in self.doc_images:
@@ -466,6 +481,25 @@ class OTCDocConvertor:
                     pathlib.Path(img_dest, img).resolve(strict=False)
                 )
 
+        context = dict(
+            title=self.args.title,
+            project=self.args.service,
+            repo_name=self.args.repo_name,
+            pdf_name=self.args.pdf_name
+        )
+        loader = FileSystemLoader([self.args.templates_location])
+        env = Environment(loader=loader, autoescape=select_autoescape())
+        for f in loader.list_templates():
+            outfile_tmpl = env.get_template(f)
+            outfile_rendered = outfile_tmpl.render(**context)
+            target_file = pathlib.Path(self.args.dest, f)
+            target_file.parent.mkdir(parents=True, exist_ok=True)
+            with open(
+                target_file, 'w', encoding='utf-8', newline=''
+            ) as out:
+                logging.debug(f"Generating {f} from template...")
+                out.write(outfile_rendered)
+
 
 def main():
     OTCDocConvertor().main()

playbooks/post.yaml

@@ -9,7 +9,7 @@
         name: "fetch_doc_artifacts"
       vars:
         fetch_doc_artifacts_name: "{{ doc.label }}"
-        fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}"
+        fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ doc.repository }}/{{ doc.project_location }}"
         # fetch_doc_artifacts_diff_patch: "{{ ansible_user_dir }}/{{ doc.label }}.patch"
         # fetch_doc_artifacts_git_patch: "{{ ansible_user_dir }}/{{ doc.label }}.git.patch"
       loop: "{{ docs }}"

playbooks/run.yaml

@@ -16,12 +16,6 @@
       register: git_log
       ignore_errors: true
 
-    - name: Configure git user name
-      command: "git config --global user.name 'OpenTelekomCloud Proposal Bot'"
-
-    - name: Configure git user email
-      command: "git config --global user.email otcbot@otc-service.com"
-
     - name: Generate RSTs
       include_role:
         name: "convert_doc"
@@ -29,13 +23,22 @@
         doc_label: "{{ doc.label }}"
         dest: "{{ ansible_user_dir }}/{{ doc.repository }}/{{ doc.project_location }}"
         source: "{{ zuul.project.src_dir }}/{{ doc.html_location }}"
-        title: "{{ doc.title | default(omit) }}"
+        doc_title: "{{ doc.title | default(omit) }}"
+        doc_service: "{{ doc.service | default(omit) }}"
+        doc_repo_name: "{{ doc.repository | default(omit) }}"
+        doc_pdf_name: "{{ [doc.doc_type, doc.service_type] | join('-') }}"
       loop: "{{ docs }}"
       loop_control:
         loop_var: "doc"
         label: "{{ doc.label }}"
       when: "git_log.stdout is search(doc.html_location) or git_log.stdout is search('otc_doc_convertor') or git_log.stdout is search('playbooks')"
 
+      #    - name: Configure git user name
+      #      command: "git config --global user.name 'OpenTelekomCloud Proposal Bot'"
+      #
+      #    - name: Configure git user email
+      #      command: "git config --global user.email otcbot@otc-service.com"
+      #
       #    - name: Restore to the previous git state
       #      command: "git checkout HEAD^1"
       #      args:

requirements.txt

@@ -1,2 +1,3 @@
 bs4
 lxml
+Jinja2

roles/convert_doc/defaults/main.yaml

@@ -1,2 +1,3 @@
+zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
 zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
 convert_params: ""

roles/convert_doc/tasks/main.yaml

@@ -8,8 +8,28 @@
 
 - name: Add title to parameters
   ansible.builtin.set_fact:
-    convert_params: "{{ convert_params }} --title '{{ title }}'"
-  when: "title is defined"
+    convert_params: "{{ convert_params }} --title '{{ doc_title }}'"
+  when: "doc_title is defined"
+
+- name: Add service to parameters
+  ansible.builtin.set_fact:
+    convert_params: "{{ convert_params }} --service '{{ doc_service }}'"
+  when: "doc_service is defined"
+
+- name: Add repo_name to parameters
+  ansible.builtin.set_fact:
+    convert_params: "{{ convert_params }} --repo-name '{{ doc_repo_name }}'"
+  when: "doc_repo_name is defined"
+
+- name: Add pdf_name to parameters
+  ansible.builtin.set_fact:
+    convert_params: "{{ convert_params }} --pdf-name {{ doc_pdf_name }}"
+  when: "doc_pdf_name is defined"
+
+- name: Add templates-location to parameters
+  ansible.builtin.set_fact:
+    convert_params: "{{ convert_params }} --templates-location {{ zuul_work_dir }}/templates"
+
 
 - name: Convert {{ doc_label | default('') }} HTML to RST
   args:

roles/fetch_doc_artifacts/tasks/doc.yaml

@@ -38,6 +38,6 @@
             url: "{{ fetch_doc_artifacts_name }}.tar.gz"
             metadata:
               type: docs_archive
-              doc_service: "{{ doc.service }}"
-              doc_type: "{{ doc.type }}"
+              doc_service: "{{ doc.service_type }}"
+              doc_type: "{{ doc.doc_type }}"
               doc_label: "{{ fetch_doc_artifacts_name }}"

roles/fetch_doc_artifacts/tasks/patch.yaml

@@ -27,6 +27,6 @@
             url: "{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch"
             metadata:
               type: "patch"
-              doc_service: "{{ doc.service }}"
-              doc_type: "{{ doc.type }}"
+              doc_service: "{{ doc.service_type }}"
+              doc_type: "{{ doc.doc_type }}"
               patch_type: "{{ patch_type }}"

templates/conf.py

@@ -0,0 +1,95 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# !!!
+# This file is generated out of template in doc-exports repository. 
+# Beware overwriting it locally.
+
+import os
+import sys
+
+extensions = [
+    'otcdocstheme'
+]
+
+otcdocs_auto_name = False
+otcdocs_auto_version = False
+
+project = '{{ project }}'
+otcdocs_repo_name = '{{ repo_name }}'
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../../'))
+sys.path.insert(0, os.path.abspath('../'))
+sys.path.insert(0, os.path.abspath('./'))
+
+# -- General configuration ----------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#
+source_encoding = 'utf-8'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+copyright = u'2022-present, Open Telekom Cloud'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+language = 'en'
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+show_authors = False
+
+# -- Options for HTML output --------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  Major themes that come with
+# Sphinx are currently 'default' and 'sphinxdoc'.
+# html_theme_path = ["."]
+html_theme = 'otcdocs'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {}
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+{% if title is defined %}
+html_title = "{{ title }}"
+{% endif %}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# -- Options for PDF output --------------------------------------------------
+latex_documents = [
+{% if pdf_name is defined -%}
+    ('index',
+     '{{ pdf_name }}.tex',
+     u'{{ title }}',
+     u'OpenTelekomCloud', 'manual'),
+{% endif -%}
+]

templates/docutils.conf

@@ -0,0 +1,2 @@
+[html writers]
+table-style: table, caption-top

tox.ini

@@ -37,4 +37,4 @@ commands = {posargs}
 #      breaks should occur before the binary operator for readability.
 ignore = H306,H4,W503
 show-source = True
-exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,templates