Copy referred images into the destination and add git patch (#7)

Copy referred images into the destination and add git patch

Include referred images in the result

Reviewed-by: OpenTelekomCloud Bot <None>

otc_doc_convertor/convertor.py

@@ -7,6 +7,7 @@ import logging
 import os
 import pathlib
 import re
+import shutil
 
 
 class OTCDocConvertor:
@@ -124,6 +125,8 @@ class OTCDocConvertor:
                     cap.name = 'figcaption'
                     figure.append(cap)
                 if img:
+                    # Store all referred images for copying
+                    self.doc_images.add(img['src'])
                     img['src'] = '/_static/images/' + img['src']
                     figure.append(img)
                 i.replace_with(figure)
@@ -233,12 +236,12 @@ class OTCDocConvertor:
             '--dest',
             help='Directory to write resulting files')
         self.args = parser.parse_args()
-        retval = os.getcwd()
         meta_data = json.loads(open(
             pathlib.Path(self.args.path, "CLASS.TXT.json")
         ).read())
         metadata_by_uri = dict()
         metadata_by_code = dict()
+        self.doc_images = set()
         if self.args.dest:
             dest = pathlib.Path(self.args.dest)
         else:
@@ -415,7 +418,16 @@ class OTCDocConvertor:
                     f" of result/{path}/index.rst")
                 p.unlink()
 
-        os.chdir(retval)
+        # Copy used images
+        if len(self.doc_images) > 0:
+            logging.debug("Processing images")
+            img_dest = pathlib.Path(dest, '_static', 'images')
+            img_dest.mkdir(parents=True, exist_ok=True)
+            for img in self.doc_images:
+                shutil.copyfile(
+                    pathlib.Path(self.args.path, img).resolve(strict=False),
+                    pathlib.Path(img_dest, img).resolve(strict=False)
+                )
 
 
 def main():

playbooks/post.yaml

@@ -1,44 +1,18 @@
 ---
-- hosts: localhost
-  vars:
-    vault_addr: "{{ zuul_vault_addr }}"
-    vault_secret_dest: "{{ zuul.executor.work_root }}/.approle-secret"
-    vault_token_dest: "{{ zuul.executor.work_root }}/.approle-token"
-
-  roles:
-    # Get the Vault token from prepared secret-id
-    - role: create-vault-approle-token
-      vault_role_id: "{{ zuul_vault.vault_role_id }}"
-      vault_wrapping_token_id: "{{ lookup('file', vault_secret_dest) }}"
-
 - hosts: all
-  vars:
-    vault_token_dest: "{{ zuul.executor.work_root }}/.approle-token"
-    vault_addr: "{{ zuul_vault_addr }}"
   tasks:
+    - name: Read project docs configuration
+      include_vars: "{{ docs_update_data_file }}"
 
-    - name: Fetch organization tokens
+    - name: Fetch generated artifacts
-      no_log: true
+      include_role:
-      check_mode: false
+        name: "fetch_doc_artifacts"
-      ansible.builtin.uri:
+      vars:
-        url: "{{ vault.vault_addr }}/v1/{{ vault.vault_token_path }}"
+        fetch_doc_artifacts_name: "{{ doc.label }}"
-        headers:
+        fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}"
-          "X-Vault-Token": "{{ lookup('file', vault_token_dest) }}"
+        fetch_doc_artifacts_diff_patch: "{{ ansible_user_dir }}/{{ doc.label }}.patch"
-        method: "POST"
+        fetch_doc_artifacts_git_patch: "{{ ansible_user_dir }}/{{ doc.label }}.git.patch"
-        body:
+      loop: "{{ docs }}"
-          org_name: "opentelekomcloud-docs"
+      loop_control:
-        body_format: "json"
+        loop_var: "doc"
-      register: "org_token"
+        label: "{{ doc.label }}"
-
-    - name: Revoke GitHub token lease
-      check_mode: false
-      no_log: true
-      uri:
-        url: "{{ vault.vault_addr }}/v1/sys/leases/revoke"
-        headers:
-          "X-Vault-Token": "{{ vault.vault_token }}"
-        method: "PUT"
-        body:
-          lease_id: "{{ org_token.json.lease_id }}"
-        body_format: "json"
-        status_code: 204

playbooks/propose_update.yaml

@@ -52,7 +52,6 @@
         - "org_token is defined"
         - "org_token.json is defined"
 
-
 - hosts: all
   vars:
     github_token_dest: "{{ zuul.executor.work_root }}/.github"

playbooks/run.yaml

@@ -1,10 +1,5 @@
 ---
 - hosts: all
-  vars:
-    docs_rst_location: "docs"
-    docs_base_location: "base"
-    docs_new_location: "new"
-
   tasks:
     - name: Read project docs configuration
       include_vars: "{{ docs_update_data_file }}"
@@ -16,6 +11,12 @@
       register: git_log
       ignore_errors: true
 
+    - name: Configure git user name
+      command: "git config --global user.name 'OpenTelekomCloud Proposal Bot'"
+
+    - name: Configure git user email
+      command: "git config --global user.email 52695153+otcbot@users.noreply.github.com"
+
     - name: Generate new RSTs
       include_role:
         name: "convert_doc"
@@ -54,22 +55,13 @@
         doc_label: "{{ doc.label }}"
         generate_doc_patch_base_location: "{{ docs_base_location }}"
         generate_doc_patch_new_location: "{{ docs_new_location }}"
+        generate_doc_patch_repository: "{{ doc.repository }}"
+        generate_doc_patch_project_location: "{{ doc.project_location }}"
         generate_doc_patch_base: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}"
-        generate_doc_patch_patch_location: "{{ ansible_user_dir }}/{{ doc.label }}.patch"
+        generate_doc_patch_diff_patch_location: "{{ ansible_user_dir }}/{{ doc.label }}.patch"
+        generate_doc_patch_git_patch_location: "{{ ansible_user_dir }}/{{ doc.label }}.git.patch"
       loop: "{{ docs }}"
       loop_control:
         loop_var: "doc"
         label: "{{ doc.label }}"
       when: "git_log.stdout is search(doc.html_location) or git_log.stdout is search('otc_doc_convertor')"
-
-    - name: Fetch generated artifacts
-      include_role:
-        name: "fetch_doc_artifacts"
-      vars:
-        fetch_doc_artifacts_name: "{{ doc.label }}"
-        fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}"
-        fetch_doc_artifacts_patch: "{{ ansible_user_dir }}/{{ doc.label }}.patch"
-      loop: "{{ docs }}"
-      loop_control:
-        loop_var: "doc"
-        label: "{{ doc.label }}"

roles/fetch_doc_artifacts/tasks/main.yaml

@@ -9,11 +9,32 @@
   when: "doc_dir.matched > 0"
   include_tasks: doc.yaml
 
-- name: Find {{ fetch_doc_artifacts_name }} Patch
+- name: Find {{ fetch_doc_artifacts_name }} diff patch
   stat:
-    path: "{{ fetch_doc_artifacts_patch }}"
+    path: "{{ fetch_doc_artifacts_diff_patch }}"
-  register: doc_patch
+  register: doc_diff_patch
+  when: "fetch_doc_artifacts_diff_patch is defined"
 
-- name: Process {{ fetch_doc_artifacts_name }} Patch file
+- name: Process {{ fetch_doc_artifacts_name }} diff patch file
-  when: "doc_patch.stat.exists"
+  when:
+    - "fetch_doc_artifacts_diff_patch is defined"
+    - "doc_diff_patch.stat.exists"
   include_tasks: patch.yaml
+  vars:
+    patch_file: "{{ fetch_doc_artifacts_diff_patch }}"
+    patch_type: "diff"
+
+- name: Find {{ fetch_doc_artifacts_name }} git patch
+  stat:
+    path: "{{ fetch_doc_artifacts_git_patch }}"
+  register: doc_git_patch
+  when: "fetch_doc_artifacts_git_patch is defined"
+
+- name: Process {{ fetch_doc_artifacts_name }} git patch file
+  when:
+    - "fetch_doc_artifacts_git_patch is defined"
+    - "doc_git_patch.stat.exists"
+  include_tasks: patch.yaml
+  vars:
+    patch_file: "{{ fetch_doc_artifacts_git_patch }}"
+    patch_type: "git"

roles/fetch_doc_artifacts/tasks/patch.yaml

@@ -1,9 +1,9 @@
 - block:
     - name: Fetch archive
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/{{ fetch_doc_artifacts_name }}.patch"
+        dest: "{{ zuul.executor.log_root }}/{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch"
         mode: pull
-        src: "{{ fetch_doc_artifacts_patch }}"
+        src: "{{ patch_file }}"
         verify_host: true
         owner: no
         group: no
@@ -12,8 +12,8 @@
 - block:
     - name: Copy archive
       copy:
-        dest: "{{ zuul_output_dir }}/logs/{{ fetch_doc_artifacts_name }}.patch"
+        dest: "{{ zuul_output_dir }}/logs/{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch"
-        src: "{{ fetch_doc_artifacts_patch }}"
+        src: "{{ patch_file }}"
         mode: 0644
         remote_src: true
   when: zuul_use_fetch_output
@@ -23,9 +23,10 @@
     data:
       zuul:
         artifacts:
-          - name: "{{ fetch_doc_artifacts_name }} patch"
+          - name: "{{ fetch_doc_artifacts_name }} {{ patch_type }} patch"
-            url: "patches/{{ fetch_doc_artifacts_name }}.patch"
+            url: "{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch"
             metadata:
-              type: docs_patch
+              type: "patch"
               doc_service: "{{ doc.service }}"
               doc_type: "{{ doc.type }}"
+              patch_type: "{{ patch_type }}"

roles/generate_doc_patch/defaults/main.yaml

@@ -0,0 +1,2 @@
+generate_doc_patch_repo_location: "{{ ansible_user_dir }}/{{ generate_doc_patch_repository }}/{{ generate_doc_patch_project_location }}"
+generate_doc_patch_doc_location: "{{ generate_doc_patch_repo_location }}/{{ generate_doc_patch_project_location }}"

zuul.yaml

@@ -28,12 +28,13 @@
     name: otc-doc-exports-convert-base
     parent: unittests
     abstract: true
-    nodeset: ubuntu-focal
+    nodeset: ubuntu-jammy
     description: |
       Convert doc exports from html to rst and generate corresponding rst diff
       files.
     pre-run: playbooks/pre.yaml
     run: playbooks/run.yaml
+    post-run: playbooks/post.yaml
 
 - job:
     name: otc-doc-exports-convert-compute
@@ -92,6 +93,10 @@
 - project:
     merge-mode: squash-merge
     default-branch: main
+    vars:
+      docs_rst_location: "docs"
+      docs_base_location: "base"
+      docs_new_location: "new"
     check:
       jobs:
         - otc-tox-pep8: