Added setuptools as requirement

Reviewed-by: tischrei <tino.schreiber@t-systems.com>
Co-authored-by: Gode, Sebastian <sebastian.gode@t-systems.com>
Co-committed-by: Gode, Sebastian <sebastian.gode@t-systems.com>

otc_metadata/data/documents/iam-permissions.yaml

@@ -0,0 +1,8 @@
+---
+html_location: docs/iam/permissions
+link: /identity-access-management/permissions/
+pdf_name: iam-permissions
+rst_location: doc/permissions/source
+service_type: iam
+title: Permissions
+type: permissions

otc_metadata/data/documents/sd-umn.yaml

@@ -0,0 +1,9 @@
+---
+hc_location: usermanual/sd
+html_location: docs/sd/umn
+link: /status-dashboard/umn/
+pdf_name: sd-umn
+rst_location: umn/source
+service_type: sd
+title: User Guide
+type: umn

otc_metadata/data/documents/wafd-api-ref.yaml

@@ -1,5 +1,4 @@
 ---
-environment: internal
 hc_location: api/dwaf
 html_location: docs/wafd/api-ref
 link: /web-application-firewall-dedicated/api-ref/

otc_metadata/data/documents/wafd-umn.yaml

@@ -1,5 +1,4 @@
 ---
-environment: internal
 hc_location: usermanual/dwaf
 html_location: docs/wafd/umn
 link: /web-application-firewall-dedicated/umn/

otc_metadata/data/service_categories/big_data.yaml

@@ -1,3 +0,0 @@
----
-name: big_data
-title: Big Data and Data Analysis

otc_metadata/data/service_categories/database.yaml

@@ -1,3 +1,3 @@
 ---
 name: database
-title: Database
+title: Databases

otc_metadata/data/services/sd.yaml

@@ -0,0 +1,16 @@
+---
+repositories:
+- environment: internal
+  repo: docs-swiss/status-dashboard
+  type: gitea
+- environment: public
+  repo: opentelekomcloud-docs-swiss/status-dashboard
+  type: github
+service_category: other
+service_title: Status Dashboard
+service_type: sd
+service_uri: status-dashboard/umn
+environment: hidden
+teams:
+- name: docs-eco-rw
+  permission: write

otc_metadata/services.py

@@ -48,6 +48,15 @@ class Services(object):
             self._service_data["services"],
             key=lambda x: f"{x.get('service_type')}{x.get('service_title')}",
         )
+        # sort service categories by <name>_<title>
+        self._service_data["service_categories"] = sorted(
+            self._service_data["service_categories"],
+            key=lambda x: f"{x.get('name')}{x.get('title')}",
+        )
+        other = {'name': 'other', 'title': 'Other'}
+        if other in self._service_data["service_categories"]:
+            self._service_data["service_categories"].remove(other)
+            self._service_data["service_categories"].append(other)
 
     def _rewrite_data(self):
         otc_metadata.data.rewrite_data("services.yaml", self._service_data)
@@ -113,6 +122,58 @@ class Services(object):
             res[cat]["docs"].append(res_doc)
         return res
 
+    def service_types_with_doc_types(self, environment=None):
+        """Retrieve type and title from services and corresponding docs.
+            As well as a list of all available doc types with title.
+
+        :param str environment: Optional service environment.
+        """
+        service_list = []
+        docs = []
+
+        for service in self.all_services:
+            if "environment" in service:
+                if service["environment"] != environment:
+                    continue
+            if not service["service_title"]:
+                continue
+            if not service["service_type"]:
+                continue
+
+            doc_list = []
+            for doc in self.all_docs:
+                if "environment" in doc:
+                    if doc["environment"] != environment:
+                        continue
+                if doc["service_type"] == service["service_type"]:
+                    doc_list.append({
+                        "title": doc["title"],
+                        "type": doc["type"]
+                    })
+
+                new_doc = {
+                    "type": doc["type"],
+                    "title": doc["title"]
+                }
+                type_exists = any(
+                    doc_dict["type"] == new_doc["type"] for doc_dict in docs
+                )
+                if not type_exists:
+                    docs.append(new_doc)
+
+            service_list.append({
+                "service_title": service["service_title"],
+                "service_type": service["service_type"],
+                "docs": doc_list
+            })
+
+        res = {
+            "services": service_list,
+            "docs": docs
+        }
+
+        return res
+
     def docs_by_service_category(self, category, environment=None):
         """List services matching category
 
@@ -176,6 +237,8 @@ class Services(object):
             srv_res = dict(
                 service_title=srv["service_title"],
                 service_type=srv["service_type"],
+                service_category=srv["service_category"],
+                service_environment=environment,
                 docs=[],
             )
             if "teams" in srv:
@@ -199,6 +262,7 @@ class Services(object):
                         rst_location=doc["rst_location"],
                         title=doc["title"],
                         type=doc.get("type", "dummy"),
+                        link=doc["link"],
                     )
                     if "pdf_name" in doc:
                         doc_res["pdf_name"] = doc["pdf_name"]

otc_metadata/templates/conf.py.j2

@@ -37,6 +37,18 @@ otcdocs_git_fqdn = '{{ git_fqdn }}'
 otcdocs_git_type = '{{ git_type }}'
 {%- endif %}
 
+# Those variables are needed for indexing into OpenSearch
+otcdocs_doc_environment = '{{ doc_environment }}'
+otcdocs_doc_link = '{{ doc_link }}'
+otcdocs_doc_title = '{{ doc_title }}'
+otcdocs_doc_type = '{{ doc_type }}'
+otcdocs_service_category = '{{ service_category }}'
+otcdocs_service_title = '{{ service_title }}'
+otcdocs_service_type = '{{ service_type }}'
+otcdocs_search_environment = 'hc_swiss'
+otcdocs_search_index = 'search_index_swiss'
+otcdocs_search_url = "https://opensearch.eco.tsi-dev.otc-service.com/"
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

otc_metadata/templates/doc_requirements.txt.j2

@@ -1,7 +1,15 @@
 sphinx>=2.0.0,!=2.1.0 # BSD
+{% if target_environment == 'public' %}
+otcdocstheme<1.0.0 # Apache-2.0
+{% elif target_environment == 'internal' %}
 otcdocstheme # Apache-2.0
+{% else %}
+otcdocstheme # Apache-2.0
+{% endif %}
 # releasenotes
 reno>=3.1.0 # Apache-2.0
 
 otc-sphinx-directives>=0.1.0
+sphinx-minify>=0.0.1 # Apache-2.0
 git+https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-swiss.git#egg=otc_metadata
+setuptools

otc_metadata/templates/tox.ini.j2

@@ -24,7 +24,9 @@ commands = {posargs}
 # This env is invoked in the periodic pipeline and is therefore responsible to
 # build all relevant docs at once.
 [testenv:docs]
-deps = -r{toxinidir}/doc/requirements.txt
+deps =
+  -r{toxinidir}/doc/requirements.txt
+  -c https://raw.githubusercontent.com/opentelekomcloud-docs-swiss/docs-constraints/main/constraints.txt
 allowlist_externals =
   mkdir
   cp
@@ -32,15 +34,16 @@ allowlist_externals =
   rm
   sphinx-build
 commands =
-  rm -rf doc/build/html doc/build/doctrees
-  sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html
+  rm -rf doc/build/html doc/build/html_temp doc/build/doctrees
+  sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html_temp
+  sphinx-minify --input-directory doc/build/html_temp/ --output-directory doc/build/html 
 {%- for doc in docs %}
   {[testenv:{{ doc.type }}]commands}
   {[testenv:json-{{ doc.type }}]commands}
 {%- endfor %}
 
 [testenv:docs-pdf]
-deps = -r{toxinidir}/doc/requirements.txt
+deps = {[testenv:docs]deps}
 allowlist_externals =
   rm
   mkdir
@@ -59,26 +62,24 @@ commands =
 {% set loc = doc.rst_location | replace('/source', '') %}
 # HTML version
 [testenv:{{ doc.type }}]
-deps = -r{toxinidir}/doc/requirements.txt
+deps = {[testenv:docs]deps}
 allowlist_externals =
   cp
   mkdir
 commands =
-  sphinx-build -W --keep-going -b html {{ loc }}/source doc/build/html/{{ doc.type }}
-{%- if doc.type == 'api-ref' %}
-  mkdir -p api-ref/build/html
-  cp -av doc/build/html/api-ref api-ref/build/html
-{%- elif doc.type == 'umn' %}
-  mkdir -p umn/build/html
-  cp -av doc/build/html/umn umn/build/html
-{%- elif doc.type == 'dev-guide' %}
+  sphinx-build -W --keep-going -b html {{ loc }}/source doc/build/html_temp/{{ doc.type }}
+  sphinx-minify --input-directory doc/build/html_temp/{{ doc.type }} --output-directory doc/build/html/{{ doc.type }}
+{%- if doc.type == 'dev-guide' %}
   mkdir -p dev_guide/build/html
   cp -av doc/build/html/dev-guide dev_guide/build/html
+{%- else %}
+  mkdir -p {{ doc.type }}/build/html
+  cp -av doc/build/html/{{ doc.type }} {{ doc.type }}/build/html
 {%- endif %}
 
 # Json version (for search)
 [testenv:json-{{ doc.type }}]
-deps = -r{toxinidir}/doc/requirements.txt
+deps = {[testenv:docs]deps}
 allowlist_externals =
   cp
   mkdir
@@ -88,21 +89,18 @@ commands =
   # Drop data useless for the search - wrap it also with sh/xargs due to bugs
   # in tox
   sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf"
-{%- if doc.type == 'api-ref' %}
-  mkdir -p api-ref/build/json
-  cp -av doc/build/json/api-ref api-ref/build/json
-{%- elif doc.type == 'umn' %}
-  mkdir -p umn/build/json
-  cp -av doc/build/json/umn umn/build/json
-{%- elif doc.type == 'dev-guide' %}
+{%- if doc.type == 'dev-guide' %}
   mkdir -p dev_guide/build/json
   cp -av doc/build/json/dev-guide dev_guide/build/json
+{%- else %}
+  mkdir -p {{ doc.type }}/build/json
+  cp -av doc/build/json/{{ doc.type }} {{ doc.type }}/build/json
 {%- endif %}
 
 {% if doc.pdf_name -%}
 # PDF version
 [testenv:{{ doc.type }}-pdf-docs]
-deps = -r{toxinidir}/doc/requirements.txt
+deps = {[testenv:docs]deps}
 allowlist_externals =
   rm
   mkdir

tools/generate_doc_confpy.py

@@ -148,8 +148,30 @@ def process_repositories(args, service):
             context["html_options"] = dict(
                 disable_search=True,
                 site_name="Internal Documentation Portal",
-                logo_url="https://docs-int.otc-service.com",
+                logo_url="https://docs-int.sc.otc.t-systems.com",
             )
+        else:
+            context["html_options"] = dict(
+                site_name="Swiss Open Telekom Cloud Docs",
+                logo_url="https://docs-beta.sc.otc.t-systems.com",
+            )
+        context["doc_environment"] = args.target_environment
+        if doc['link']:
+            context["doc_link"] = doc['link']
+        else:
+            context["doc_link"] = (
+                '/'
+                + service['service_uri']
+                + '/'
+                + doc['type']
+                + '/'
+            )
+        context["doc_title"] = doc['title']
+        context["doc_type"] = doc['type']
+        context["service_category"] = service['service_category']
+        context["service_title"] = service['service_title']
+        context["service_type"] = service['service_type']
+
         conf_py_content = conf_py_template.render(**context)
         with open(conf_py_path, "w", encoding="utf-8", newline="") as out:
             logging.debug(f"Generating {conf_py_path} from template...")
@@ -167,6 +189,8 @@ def process_repositories(args, service):
             title=f"{service['service_title']} - Service Based View",
             service_type=service["service_type"]
         )
+        context["service_category"] = service['service_category']
+        context["service_title"] = service['service_title']
         if not copy_path.exists():
             logging.info("Path for sbv does not exist")
             copy_path.mkdir(parents=True, exist_ok=True)
@@ -179,7 +203,12 @@ def process_repositories(args, service):
             context["html_options"] = dict(
                 disable_search=True,
                 site_name="Internal Documentation Portal",
-                logo_url="https://docs-int.otc-service.com",
+                logo_url="https://docs-int.sc.otc.t-systems.com",
+            )
+        else:
+            context["html_options"] = dict(
+                site_name="Swiss Open Telekom Cloud Docs",
+                logo_url="https://docs-beta.sc.otc.t-systems.com",
             )
         sbv_title = (service["service_title"] + "\n"
                      + ('=' * len(service["service_title"])))
@@ -218,6 +247,8 @@ def process_repositories(args, service):
                 doc["type"] = "dev-guide"
             context["docs"].append(doc)
 
+        context["target_environment"] = args.target_environment
+
         tox_ini_content = tox_ini_template.render(**context)
         tox_ini_path = pathlib.Path(copy_to, "tox.ini")
         doc_requirements_content = doc_requirements_template.render(**context)
@@ -248,7 +279,7 @@ def process_repositories(args, service):
         args.commit_description
     )
     push_args = ["--set-upstream", "origin", branch_name]
-    if args.branch_force:
+    if args.force_push:
         push_args.append("--force")
     repo_to.git.push(*push_args)
     if "github" in url_to:
@@ -294,6 +325,7 @@ def main():
     parser.add_argument(
         "--target-environment",
         required=True,
+        choices=["internal", "public"],
         help="Environment to be used as a source",
     )
     parser.add_argument("--service-type", help="Service to update")
@@ -328,6 +360,11 @@ def main():
         help=("Whether to overwrite index.rst for service-based-view."
               + "\nCan only be used if --update-sbv is also specified")
     )
+    parser.add_argument(
+        "--force-push",
+        action="store_true",
+        help="Whether to force push the commit"
+    )
     parser.add_argument(
         "--commit-description",
         default=(

tools/generate_docexports_data.py

@@ -10,6 +10,7 @@ import otc_metadata.services
 
 def main():
     data = otc_metadata.services.Services()
+    data._sort_data()
 
     _yaml = YAML()
     _yaml.indent(mapping=2, sequence=4, offset=2)