add zuul jobs (#1)

Enable zuul jobs

Reviewed-by: OpenTelekomCloud Bot <None>
Reviewed-by: Vladimir Hasko <vladimirhasko@gmail.com>

.gitignore

@@ -1,3 +1,6 @@
+AUTHORS
+ChangeLog
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,2 +1,32 @@
 # doc-exports
 Repository to track document exports, not for public use
+
+Document sources (in HTML) are placed under docs/<service_name>/<doc_type>
+
+In the there are files docs_<service_category>.yaml that
+describe required data for the particular document. For
+all of the documents configured in those files conversion
+from HTML to RST will be attempted. In addition to that a
+patch file (changes in the RST format) will be generated.
+
+## Automatic pull request proposal
+
+Change for any of the documents configured in the
+docs_<service_category>.yaml would trigger job to propose
+corresponding changes to the target repository. This is
+implemented in the following way:
+
+- HTML content from the pull request will be converted
+  with the latest conversion script (otc_doc_convertor) to
+RST (new)
+
+- HTML content for the same document from current main
+  branch with the corresponding version of the conversion
+script will be converted to RST (base).
+
+- **New** state is compared against **base** and
+  corresponding patch file is generated
+
+- Resulting patch file is force-applied in the target
+  repository (skipping all conflicts) and PR is created
+

docs_big-data.yaml

@@ -0,0 +1,14 @@
+---
+docs:
+  - label: ma_umn
+    service: ma
+    type: umn
+    html_location: docs/modelarts/umn
+    repository: opentelekomcloud-docs/modelarts
+    project_location: /umn/source
+  - label: ma_api
+    service: ma
+    type: api-ref
+    html_location: docs/modelarts/api-ref
+    repository: opentelekomcloud-docs/modelarts
+    project_location: /api-ref/source

docs_compute.yaml

@@ -0,0 +1,20 @@
+---
+docs:
+  - label: ecs_umn
+    service: ecs
+    type: umn
+    html_location: docs/ecs/umn
+    repository: opentelekomcloud-docs/elastic-cloud-server
+    project_location: /umn/source
+  - label: ecs_api
+    service: ecs
+    type: api-ref
+    html_location: docs/ecs/api-ref
+    repository: opentelekomcloud-docs/elastic-cloud-server
+    project_location: /api-ref/source
+  - label: ecs_dev
+    service: ecs
+    type: dev-guide
+    html_location: docs/ecs/dev
+    repository: opentelekomcloud-docs/elastic-cloud-server
+    project_location: /dev_guide/source

otc_doc_convertor/convertor.py

@@ -179,22 +179,22 @@ class OTCDocConvertor:
         for li in soup.body.find_all("li"):
             del li['id']
 
-        # for pre in soup.body.find_all("pre"):
+        for pre in soup.body.find_all("pre"):
-        #     text = pre.get_text()
+            text = pre.get_text()
-        #     # if text.startswith("{"):
+            # if text.startswith("{"):
-        #     #    pre["class"] = "data"
+            #    pre["class"] = "data"
-        #     if re.search(
+            if re.search(
-        #         r'\[[a-z]*@\w+.*\][\s#>]?',
+                r'\[[a-z]*@\w+.*\][\s#>]?',
-        #         text
+                text
-        #     ):
+            ):
-        #         # Something like "[root@ecs-test-0001 ~]#"
+                # Something like "[root@ecs-test-0001 ~]#"
-        #         pre["class"] = "console"
+                pre["class"] = "console"
-        #     elif re.match(
+            elif re.match(
-        #         r'^(GET|PUT|POST|DELETE)',
+                r'^(GET|PUT|POST|DELETE)',
-        #         text
+                text
-        #     ):
+            ):
-        #         # Something like "DELETE https://some_url"
+                # Something like "DELETE https://some_url"
-        #         pre["class"] = "text"
+                pre["class"] = "text"
 
         # And now specialities
         rawize_strings = [
@@ -344,7 +344,7 @@ class OTCDocConvertor:
                  open(pathlib.Path(dest, target_path,
                       f"{target}.rst"), 'w') as writer:
                 logging.info(f"Post processing {target}")
-                # writer.write(f":original_name: {f.name}\n\n")
+                writer.write(f":original_name: {f.name}\n\n")
                 # Add root file label
                 writer.write(f".. _{f.name.replace('.html', '')}:\n\n")
                 # post process some usual stuff
@@ -363,15 +363,15 @@ class OTCDocConvertor:
                     processed_line = re.sub(
                         r'.. code:: screen$',
                         r'.. code-block::', processed_line)
-                    # for lexer in ["json", "bash", "text", "console"]:
+                    for lexer in ["json", "bash", "text", "console"]:
-                    #     processed_line = re.sub(
+                        processed_line = re.sub(
-                    #         f".. code:: {lexer}$",
+                            f".. code:: {lexer}$",
-                    #         f".. code-block:: {lexer}", processed_line)
+                            f".. code-block:: {lexer}", processed_line)
-                    #     if re.match(rf".. code:: {lexer}\s", processed_line):
+                        if re.match(rf".. code:: {lexer}\s", processed_line):
-                    #         logging.error(
+                            logging.error(
-                    #             f"'code-block: {lexer}' with something "
+                                f"'code-block: {lexer}' with something "
-                    #             "afterwards")
+                                "afterwards")
-                    #         exit(1)
+                            exit(1)
                     # spaces are important, since code-block may reside inside
                     # of the cell
                     processed_line = re.sub(

playbooks/post.yaml

@@ -0,0 +1,44 @@
+---
+- hosts: localhost
+  vars:
+    vault_addr: "{{ zuul_vault_addr }}"
+    vault_secret_dest: "{{ zuul.executor.work_root }}/.approle-secret"
+    vault_token_dest: "{{ zuul.executor.work_root }}/.approle-token"
+
+  roles:
+    # Get the Vault token from prepared secret-id
+    - role: create-vault-approle-token
+      vault_role_id: "{{ zuul_vault.vault_role_id }}"
+      vault_wrapping_token_id: "{{ lookup('file', vault_secret_dest) }}"
+
+- hosts: all
+  vars:
+    vault_token_dest: "{{ zuul.executor.work_root }}/.approle-token"
+    vault_addr: "{{ zuul_vault_addr }}"
+  tasks:
+
+    - name: Fetch organization tokens
+      no_log: true
+      check_mode: false
+      ansible.builtin.uri:
+        url: "{{ vault.vault_addr }}/v1/{{ vault.vault_token_path }}"
+        headers:
+          "X-Vault-Token": "{{ lookup('file', vault_token_dest) }}"
+        method: "POST"
+        body:
+          org_name: "opentelekomcloud-docs"
+        body_format: "json"
+      register: "org_token"
+
+    - name: Revoke GitHub token lease
+      check_mode: false
+      no_log: true
+      uri:
+        url: "{{ vault.vault_addr }}/v1/sys/leases/revoke"
+        headers:
+          "X-Vault-Token": "{{ vault.vault_token }}"
+        method: "PUT"
+        body:
+          lease_id: "{{ org_token.json.lease_id }}"
+        body_format: "json"
+        status_code: 204

playbooks/pre.yaml

@@ -0,0 +1,13 @@
+---
+- hosts: all
+  roles:
+    - ensure-python
+    - ensure-virtualenv
+    - ensure-gh
+  tasks:
+    - name: Install convertor
+      pip:
+        chdir: "{{ zuul.project.src_dir }}"
+        virtualenv: "{{ ansible_user_dir }}/.venv"
+        name: .
+        editable: "yes"

playbooks/run.yaml

@@ -0,0 +1,75 @@
+---
+- hosts: all
+  vars:
+    docs_rst_location: "docs"
+    docs_base_location: "base"
+    docs_new_location: "new"
+
+  tasks:
+    - name: Read project docs configuration
+      include_vars: "{{ docs_update_data_file }}"
+
+    - name: Detect list of changes
+      command: "git log -1 --name-only --pretty="
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      register: git_log
+      ignore_errors: true
+
+    - name: Generate new RSTs
+      include_role:
+        name: "convert_doc"
+      vars:
+        doc_label: "{{ doc.label }}"
+        dest: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}"
+        source: "{{ zuul.project.src_dir }}/{{ doc.html_location }}"
+      loop: "{{ docs }}"
+      loop_control:
+        loop_var: "doc"
+        label: "{{ doc.label }}"
+      when: "git_log.stdout is search(doc.html_location) or git_log.stdout is search('otc_doc_convertor')"
+
+    - name: Restore to the previous git state
+      command: "git checkout HEAD^1"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Generate base RSTs
+      include_role:
+        name: "convert_doc"
+      vars:
+        doc_label: "{{ doc.label }}"
+        dest: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_base_location }}"
+        source: "{{ zuul.project.src_dir }}/{{ doc.html_location }}"
+      loop: "{{ docs }}"
+      loop_control:
+        loop_var: "doc"
+        label: "{{ doc.label }}"
+      when: "git_log.stdout is search(doc.html_location) or git_log.stdout is search('otc_doc_convertor')"
+
+    - name: Generate patch file
+      include_role:
+        name: "generate_doc_patch"
+      vars:
+        doc_label: "{{ doc.label }}"
+        generate_doc_patch_base_location: "{{ docs_base_location }}"
+        generate_doc_patch_new_location: "{{ docs_new_location }}"
+        generate_doc_patch_base: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}"
+        generate_doc_patch_patch_location: "{{ ansible_user_dir }}/{{ doc.label }}.patch"
+      loop: "{{ docs }}"
+      loop_control:
+        loop_var: "doc"
+        label: "{{ doc.label }}"
+      when: "git_log.stdout is search(doc.html_location) or git_log.stdout is search('otc_doc_convertor')"
+
+    - name: Fetch generated artifacts
+      include_role:
+        name: "fetch_doc_artifacts"
+      vars:
+        fetch_doc_artifacts_name: "{{ doc.label }}"
+        fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}"
+        fetch_doc_artifacts_patch: "{{ ansible_user_dir }}/{{ doc.label }}.patch"
+      loop: "{{ docs }}"
+      loop_control:
+        loop_var: "doc"
+        label: "{{ doc.label }}"

playbooks/vars/docs_big-data.yaml

@@ -0,0 +1 @@
+../../docs_big-data.yaml

playbooks/vars/docs_compute.yaml

@@ -0,0 +1 @@
+../../docs_compute.yaml

process_links.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import argparse
-import subprocess
-
-
-def main():
-    parser = argparse.ArgumentParser(description='Process links.')
-    parser.add_argument(
-            'path', type=str, help='path to the files')
-    args = parser.parse_args()
-    matrix = json.loads(open("matrix.json").read())
-    for k, v in matrix.items():
-        replace = v.replace('/', '\/')
-        subprocess.run(
-            f"find {args.path} -name *'.rst' -type f -print0 | xargs"
-            f" -0 sed -i '' 's/{k}/{replace}/g'",
-            shell=True
-        )
-        print(k, v)
-
-
-if __name__ == "__main__":
-    main()

roles/apply_doc_patch/defaults/main.yaml

@@ -0,0 +1,6 @@
+apply_doc_patch_gh: "/usr/local/bin/gh"
+apply_doc_patch_branch_name: "import-{{ doc_label }}-{{ zuul.change }}"
+apply_doc_patch_pr_title: |
+  Changes to {{ doc_label }} from doc-exports#{{ zuul.change }}
+apply_doc_patch_pr_body_file: "{{ ansible_user_dir }}/{{ doc_label }}_pr_body.txt"
+apply_doc_patch_pr_label: "autoPR"

roles/apply_doc_patch/templates/pr_body.txt.j2

@@ -0,0 +1,9 @@
+This is an automatically created Pull Request for changes to {{ doc_label }} in opentelekomcloud-docs/doc-exports#{{ zuul.change }}.
+
+Please do not edit it manually, since update to the original PR will overwrite local changes.
+
+Original patch file, as well as complete rst archive,  can be found in the artifacts of the opentelekomcloud-docs/doc-exports#{{ zuul.change }}
+
+{% if rej_files.matched > 0 %}
+There were patch conflicts, please review original patch file.
+{% endif %}

roles/convert_doc/defaults/main.yaml

@@ -0,0 +1 @@
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"

roles/convert_doc/tasks/main.yaml

@@ -0,0 +1,4 @@
+- name: Convert {{ doc_label | default('') }} HTML to RST
+  args:
+    executable: "/bin/bash"
+  shell: "source {{ zuul_work_virtualenv }}/bin/activate; otc-convert-doc --dest {{ dest }} {{ source }}"

roles/fetch_doc_artifacts/defaults/main.yaml

@@ -0,0 +1,3 @@
+zuul_executor_dest: "{{ zuul.executor.log_root }}"
+zuul_output_dir: "{{ ansible_user_dir }}/zuul-output"
+zuul_use_fetch_output: "{{ zuul_site_use_fetch_output|default(false) }}"

roles/fetch_doc_artifacts/tasks/doc.yaml

@@ -0,0 +1,43 @@
+- name: Create temporary archive file
+  tempfile:
+    state: file
+    suffix: ".tar.gz"
+  register: doc_archive
+
+- name: Archive Doc
+  command: "tar -f {{ doc_archive.path }} -C {{ fetch_doc_artifacts_rst_source }} -cz ."
+  args:
+    warn: false
+
+- block:
+    - name: Fetch archive
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/{{ fetch_doc_artifacts_name }}.tar.gz"
+        mode: pull
+        src: "{{ doc_archive.path }}"
+        verify_host: true
+        owner: no
+        group: no
+  when: not zuul_use_fetch_output
+
+- block:
+    - name: Copy archive
+      copy:
+        dest: "{{ zuul_output_dir }}/logs/{{ fetch_doc_artifacts_name }}.tar.gz"
+        src: "{{ doc_archive.path }}"
+        mode: 0644
+        remote_src: true
+  when: zuul_use_fetch_output
+
+- name: Return artifact to Zuul
+  zuul_return:
+    data:
+      zuul:
+        artifacts:
+          - name: "{{ fetch_doc_artifacts_name }} archive"
+            url: "{{ fetch_doc_artifacts_name }}.tar.gz"
+            metadata:
+              type: docs_archive
+              doc_service: "{{ doc.service }}"
+              doc_type: "{{ doc.type }}"
+              doc_label: "{{ fetch_doc_artifacts_name }}"

roles/fetch_doc_artifacts/tasks/main.yaml

@@ -0,0 +1,19 @@
+- name: Inspect {{ fetch_doc_artifacts_name }} build directory
+  find:
+    file_type: any
+    follow: true
+    paths: "{{ fetch_doc_artifacts_rst_source }}"
+  register: doc_dir
+
+- name: Process {{ fetch_doc_artifacts_name }} archive
+  when: "doc_dir.matched > 0"
+  include_tasks: doc.yaml
+
+- name: Find {{ fetch_doc_artifacts_name }} Patch
+  stat:
+    path: "{{ fetch_doc_artifacts_patch }}"
+  register: doc_patch
+
+- name: Process {{ fetch_doc_artifacts_name }} Patch file
+  when: "doc_patch.stat.exists"
+  include_tasks: patch.yaml

roles/fetch_doc_artifacts/tasks/patch.yaml

@@ -0,0 +1,31 @@
+- block:
+    - name: Fetch archive
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/{{ fetch_doc_artifacts_name }}.patch"
+        mode: pull
+        src: "{{ fetch_doc_artifacts_patch }}"
+        verify_host: true
+        owner: no
+        group: no
+  when: not zuul_use_fetch_output
+
+- block:
+    - name: Copy archive
+      copy:
+        dest: "{{ zuul_output_dir }}/logs/{{ fetch_doc_artifacts_name }}.patch"
+        src: "{{ fetch_doc_artifacts_patch }}"
+        mode: 0644
+        remote_src: true
+  when: zuul_use_fetch_output
+
+- name: Return artifact to Zuul
+  zuul_return:
+    data:
+      zuul:
+        artifacts:
+          - name: "{{ fetch_doc_artifacts_name }} patch"
+            url: "patches/{{ fetch_doc_artifacts_name }}.patch"
+            metadata:
+              type: docs_patch
+              doc_service: "{{ doc.service }}"
+              doc_type: "{{ doc.type }}"

roles/generate_doc_patch/tasks/main.yaml

@@ -0,0 +1,21 @@
+---
+- name: Check base state for {{ doc_label }}
+  stat:
+    path: "{{ generate_doc_patch_base }}/{{ generate_doc_patch_base_location }}"
+  register: base_stat
+
+- name: Check new state for {{ doc_label }}
+  stat:
+    path: "{{ generate_doc_patch_base }}/{{ generate_doc_patch_new_location }}"
+  register: new_stat
+
+- name: Generate {{ doc_label }} patch file
+  shell: "diff -ruN {{ generate_doc_patch_base_location }} {{ generate_doc_patch_new_location }} > {{ generate_doc_patch_patch_location }}"
+  args:
+    chdir: "{{ generate_doc_patch_base }}"
+  register: diff_cmd
+  failed_when: "diff_cmd.rc >= 2"
+  changed_when: "diff_cmd.rc == 1"
+  when:
+    - "base_stat.stat.exists"
+    - "new_stat.stat.exists"

tox.ini

@@ -0,0 +1,40 @@
+[tox]
+minversion = 3.6
+envlist = pep8
+skipsdist = True
+ignore_basepython_conflict = True
+
+[testenv]
+usedevelop = True
+install_command = pip install {opts} {packages}
+setenv =
+    VIRTUAL_ENV={envdir}
+    LANG=en_US.UTF-8
+    LANGUAGE=en_US:en
+    LC_ALL=C
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+
+[testenv:pep8]
+commands =
+    flake8
+
+[testenv:venv]
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+commands = {posargs}
+
+[flake8]
+# The following are ignored on purpose. It's not super worth it to fix them.
+# However, if you feel strongly about it, patches will be accepted to fix them
+# if they fix ALL of the occurances of one and only one of them.
+# H306 Is about alphabetical imports - there's a lot to fix.
+# H4 Are about docstrings and there's just a huge pile of pre-existing issues.
+# W503 Is supposed to be off by default but in the latest pycodestyle isn't.
+#      Also, both openstacksdk and Donald Knuth disagree with the rule. Line
+#      breaks should occur before the binary operator for readability.
+ignore = H306,H4,W503
+show-source = True
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build