From ba47f9b9ed6a72c81195482c5e5fedccad9223a9 Mon Sep 17 00:00:00 2001 From: "Gode, Sebastian" Date: Wed, 25 Feb 2026 14:16:34 +0000 Subject: [PATCH 1/7] Make ESW public (#77) Reviewed-on: https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-rework/pulls/77 Reviewed-by: Tino Schreiber Co-authored-by: Gode, Sebastian Co-committed-by: Gode, Sebastian --- otc_metadata/data/services/esw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otc_metadata/data/services/esw.yaml b/otc_metadata/data/services/esw.yaml index a1cbfdc..824663a 100644 --- a/otc_metadata/data/services/esw.yaml +++ b/otc_metadata/data/services/esw.yaml @@ -8,5 +8,5 @@ teams: permission: write cloud_environments: - name: eu_de - visibility: internal + visibility: public is_global: false From 6e50b23d94601d358c90b259f3147083c079ff33 Mon Sep 17 00:00:00 2001 From: lbelejka Date: Wed, 25 Feb 2026 14:39:12 +0000 Subject: [PATCH 2/7] public EPS to Helpcenter (#78) Reviewed-on: https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-rework/pulls/78 Reviewed-by: Tino Schreiber Co-authored-by: lbelejka Co-committed-by: lbelejka --- otc_metadata/data/documents/eps-api-ref.yaml | 2 +- otc_metadata/data/documents/eps-umn.yaml | 2 +- otc_metadata/data/services/eps.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/otc_metadata/data/documents/eps-api-ref.yaml b/otc_metadata/data/documents/eps-api-ref.yaml index 4c5a953..32a8caf 100644 --- a/otc_metadata/data/documents/eps-api-ref.yaml +++ b/otc_metadata/data/documents/eps-api-ref.yaml @@ -8,6 +8,6 @@ title: API Reference type: api-ref cloud_environments: - name: eu_de - visibility: internal + visibility: public pdf_visibility: public pdf_enabled: true diff --git a/otc_metadata/data/documents/eps-umn.yaml b/otc_metadata/data/documents/eps-umn.yaml index cc025ca..8b9fa4b 100644 --- a/otc_metadata/data/documents/eps-umn.yaml +++ b/otc_metadata/data/documents/eps-umn.yaml @@ -8,6 +8,6 @@ title: User Guide type: umn cloud_environments: - name: eu_de - visibility: internal + visibility: public pdf_visibility: public pdf_enabled: true diff --git a/otc_metadata/data/services/eps.yaml b/otc_metadata/data/services/eps.yaml index dfe4b34..12e8329 100644 --- a/otc_metadata/data/services/eps.yaml +++ b/otc_metadata/data/services/eps.yaml @@ -1,5 +1,5 @@ --- -service_category: security-services +service_category: md service_title: Enterprise Project Service service_type: eps service_uri: enterprise-project-service @@ -8,5 +8,5 @@ teams: permission: write cloud_environments: - name: eu_de - visibility: internal + visibility: public is_global: false From 2b2df92bcf37237782be21a20e7556c14405b284 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Wed, 4 Mar 2026 10:27:39 +0000 Subject: [PATCH 3/7] LLM Metadata generation script (#79) Reviewed-on: https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-rework/pulls/79 Reviewed-by: Tino Schreiber Co-authored-by: Sebastian Gode Co-committed-by: Sebastian Gode --- tools/generate_meta.py | 459 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100755 tools/generate_meta.py diff --git a/tools/generate_meta.py b/tools/generate_meta.py new file mode 100755 index 0000000..71bbcff --- /dev/null +++ b/tools/generate_meta.py @@ -0,0 +1,459 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import base64 +import logging +import pathlib +import requests +import subprocess + +from git import exc +from git import Repo + +import otc_metadata.services + +data = otc_metadata.services.Services() + +api_session = requests.Session() + + +def extract_description(result): + """Extract description from API response and clean it.""" + if "choices" in result and len(result["choices"]) > 0: + message = result["choices"][0].get("message", {}) + description = message.get("content", "") + elif "response" in result: + description = result["response"].strip() + elif isinstance(result, dict) and "text" in result: + description = result["text"].strip() + else: + return None + + description = description.strip() + + if not description or description.isspace(): + return None + + # Extract only the first sentence + parts = description.split(".") + first_sentence = parts[0].strip() + "." + if len(first_sentence) <= 1: + first_sentence = description[:160].strip() + "." + if len(first_sentence) > 160: + first_sentence = first_sentence[:157] + "..." + return first_sentence + + +def extract_keywords(result): + """Extract keywords from API response and clean it.""" + if "choices" in result and len(result["choices"]) > 0: + message = result["choices"][0].get("message", {}) + keywords_text = message.get("content", "") + elif "response" in result: + keywords_text = result["response"].strip() + elif isinstance(result, dict) and "text" in result: + keywords_text = result["text"].strip() + else: + return None + + keywords_text = keywords_text.strip() + + if not keywords_text or keywords_text.isspace(): + return None + + keywords = [kw.strip() for kw in keywords_text.split(",")] + keywords = [kw for kw in keywords if kw and len(kw) > 0] + + keywords = keywords[:5] + + return ", ".join(keywords) + + +def generate_description_with_llm(text, service_title, llm_api_url, model_name, api_username, api_password): + """Generate a meta description using the llama.cpp /completion endpoint with up to 3 retries.""" + content_preview = text[:2000].replace("\n", " ") + prompt = ( + f"Generate a meta description (40-160 chars) for: {service_title}." + f"Content preview: {content_preview}." + f"Output ONLY the description text, nothing else." + ) + + headers = {"Content-Type": "application/json"} + if api_username and api_password: + credentials = f"{api_username}:{api_password}" + encoded_credentials = base64.b64encode(credentials.encode()).decode() + headers["Authorization"] = f"Basic {encoded_credentials}" + + for attempt in range(3): + try: + response = requests.post( + llm_api_url, + json={ + "messages": [ + {"role": "user", "content": prompt}, + ], + "model": model_name, + "temperature": 0.5, + + "top_k": 40, + "top_p": 0.9, + "min_p": 0.05, + + "repeat_last_n": 256, + "repeat_penalty": 1.18, + "presence_penalty": 0.2, + "frequency_penalty": 0.2, + + "dry_multiplier": 0.8, + "dry_base": 1.75, + "dry_allowed_length": 2, + "dry_penalty_last_n": -1, + "chat_template_kwargs": {"enable_thinking": False}, + }, + headers=headers, + timeout=15, + ) + response.raise_for_status() + result = response.json() + description = extract_description(result) + if description: + return description + logging.warning(f"Attempt {attempt + 1}: Empty or invalid response from LLM API.") + except requests.exceptions.RequestException as e: + logging.warning(f"Attempt {attempt + 1}: LLM API request failed: {e}. Retrying...") + except (KeyError, ValueError, IndexError) as e: + logging.warning(f"Attempt {attempt + 1}: LLM API response parsing failed: {e}. Retrying...") + + # After all retries failed, use fallback - extract first headline + logging.warning("All LLM API retries failed. Using fallback description from first headline.") + lines = text.split("\n") + for i, line in enumerate(lines): + line_stripped = line.strip() + if line_stripped and not line_stripped.startswith("-") and not line_stripped.startswith("#"): + # Check if next line is a headline underline (=== or ---) + if i + 1 < len(lines): + next_line = lines[i + 1].strip() + if next_line and all(c in "=-" for c in next_line): + description = line_stripped + if len(description) > 160: + description = description[:157] + "..." + return description + + return f"{service_title} documentation" + + +def generate_keywords_with_llm(text, service_title, llm_api_url, model_name, api_username, api_password): + """Generate keywords using the llama.cpp /completion endpoint with up to 3 retries.""" + content_preview = text[:2000].replace("\n", " ") + prompt = ( + f"Generate up to 5 keywords (comma-separated) for: {service_title}. " + f"Content preview: {content_preview}. " + f"Output ONLY comma-separated keywords, nothing else." + ) + + headers = {"Content-Type": "application/json"} + if api_username and api_password: + credentials = f"{api_username}:{api_password}" + encoded_credentials = base64.b64encode(credentials.encode()).decode() + headers["Authorization"] = f"Basic {encoded_credentials}" + + for attempt in range(3): + try: + response = requests.post( + llm_api_url, + json={ + "messages": [ + {"role": "user", "content": prompt}, + ], + "model": model_name, + "temperature": 0.7, + + "top_k": 40, + "top_p": 0.9, + "min_p": 0.05, + + "repeat_last_n": 256, + "repeat_penalty": 1.18, + "presence_penalty": 0.2, + "frequency_penalty": 0.2, + + "dry_multiplier": 0.8, + "dry_base": 1.75, + "dry_allowed_length": 2, + "dry_penalty_last_n": -1, + "chat_template_kwargs": {"enable_thinking": False}, + }, + headers=headers, + timeout=15, + ) + response.raise_for_status() + result = response.json() + keywords = extract_keywords(result) + if keywords: + return keywords + logging.warning(f"Attempt {attempt + 1}: Empty or invalid response from LLM API for keywords.") + except requests.exceptions.RequestException as e: + logging.warning(f"Attempt {attempt + 1}: LLM API request failed: {e}. Retrying...") + except (KeyError, ValueError, IndexError) as e: + logging.warning(f"Attempt {attempt + 1}: LLM API response parsing failed: {e}. Retrying...") + + logging.warning("All LLM API retries failed for keywords. Using fallback.") + return f"{service_title.replace('-', ' ').title()}" + + +def read_rst_content(file_path): + """Read and return the content of an RST file.""" + with open(file_path, "r", encoding="utf-8") as f: + return f.read() + + +def add_sphinx_metadata(file_path, meta_description, meta_keywords=None): + """Add Sphinx-compatible meta block at the end of an RST file.""" + content = read_rst_content(file_path) + + meta_block = "\n\n.. meta::\n" + if meta_description: + meta_block += " :description: {}\n".format(meta_description) + if meta_keywords: + meta_block += " :keywords: {}\n".format(meta_keywords) + + # Check if meta block already exists + if ".. meta::" in content: + logging.debug(f"Meta block already exists in {file_path}. Skipping.") + return False + + # Append meta block at the end of the file + new_content = content.rstrip() + meta_block + + with open(file_path, "w", encoding="utf-8", newline="") as f: + f.write(new_content) + + return True + + +def process_service(args, service): + """Process a single service and add metadata to its RST files.""" + logging.debug(f"Processing service {service['service_title']}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + repo_url = None + repo_dir = None + git_repo = None + error_list = [] + + repo = None + for r in service["repositories"]: + if r["cloud_environments"][0] == args.cloud_environment: + repo_dir = workdir / r["type"] / r["repo"] + + if r["environment"] == args.target_environment: + repo = r + break + else: + logging.debug(f"Skipping repository {r}") + continue + + if not repo_dir: + logging.info(f"No repository found for service {service['service_title']}") + return + + if repo_dir.exists(): + logging.debug(f"Repository {repo_dir} already exists") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + git_repo.heads.main.checkout() + git_repo.remotes.origin.pull() + except exc.InvalidGitRepositoryError: + logging.error("Existing repository checkout is bad") + import shutil + shutil.rmtree(repo_dir) + git_repo = None + except Exception as e: + error_list.append({"error": e, "repo": repo["repo"]}) + + if not repo_dir.exists() or git_repo is None: + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + error_list.append({"error": f"Repository type {repo['type']} is not supported", "repo": repo["repo"]}) + return + + try: + logging.debug(f"Cloning repository {repo_url}") + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception as e: + logging.error(f"Error cloning repository {repo_url}: {e}") + error_list.append({"error": f"Error cloning repository {repo_url}", "repo": repo["repo"]}) + return + + branch_name = f"add-meta-{args.branch_name}" + + try: + new_branch = git_repo.create_head(branch_name, "main") + except Exception as e: + logging.warning(f"Skipping service {service} due to {e}") + error_list.append({"error": e, "repo": repo["repo"]}) + return + + new_branch.checkout() + + rst_files = (list(repo_dir.rglob("doc/**/*.rst")) + + list(repo_dir.rglob("umn/**/*.rst")) + + list(repo_dir.rglob("api-ref/**/*.rst"))) + + processed_count = 0 + updated_count = 0 + + for rst_file in rst_files: + + logging.debug(f"Analyzing document {rst_file}") + + try: + content = read_rst_content(rst_file) + description = generate_description_with_llm( + content, + service["service_title"], + args.llm_api_url, + args.llm_model, + args.llm_username, + args.llm_password + ) + keywords = generate_keywords_with_llm( + content, + service["service_title"], + args.llm_api_url, + args.llm_model, + args.llm_username, + args.llm_password + ) + + if add_sphinx_metadata(rst_file, description, keywords): + updated_count += 1 + logging.info(f"Added meta description and keywords to {rst_file}") + else: + processed_count += 1 + + git_repo.index.add([str(rst_file)]) + + except Exception as e: + logging.error(f"Error processing {rst_file}: {e}") + error_list.append({"error": e, "repo": str(rst_file)}) + + if len(git_repo.index.diff("HEAD")) == 0: + logging.debug("No changes required for service %s", service["service_type"]) + return + + git_repo.index.commit(args.commit_description) + + try: + git_repo.git.push("--set-upstream", "origin", branch_name) + logging.info(f"Pushed changes for service {service['service_title']}") + except Exception as e: + error_list.append({"error": e, "repo": repo["repo"]}) + + if repo_url and "github" in repo_url: + subprocess.run( + args=["gh", "pr", "create", "-f"], cwd=repo_dir, check=False + ) + elif repo_url and "gitea" in repo_url and args.token: + pass + + if len(error_list) != 0: + logging.error("The following errors have happened:") + logging.error(error_list) + + logging.info(f"Processed {processed_count} files, updated {updated_count} files") + + +def main(): + parser = argparse.ArgumentParser( + description="Add Sphinx meta blocks to RST files using LLM-generated descriptions." + ) + parser.add_argument( + "--target-environment", + required=True, + choices=["internal", "public"], + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="meta-generation", + help="Branch name to be used for changes.", + ) + parser.add_argument("--token", metavar="token", help="API token") + parser.add_argument( + "--llm-api-url", + default="http://localhost:8080/v1/chat/completions", + help="URL of the LLM API server. Default: http://localhost:8080/v1/chat/completions", + ) + parser.add_argument( + "--llm-model", + default="llama2", + help="LLM model name to use. Default: llama2", + ) + parser.add_argument( + "--llm-username", + help="Username for Basic Authentication with LLM server", + ) + parser.add_argument( + "--llm-password", + help="Password for Basic Authentication with LLM server", + ) + parser.add_argument( + "--commit-description", + default=( + "Add Sphinx meta blocks to RST files\n\n" + "Generated by otc-metadata-rework/tools/generate_meta.py" + ), + help="Commit description for the commit", + ) + parser.add_argument( + "--cloud-environment", + required=True, + default="eu_de", + help="Cloud Environment. Default: eu_de", + ) + + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG) + + services = [] + if args.service_type: + services = [data.get_service_with_repo_by_service_type(service_type=args.service_type)] + else: + services = data.services_with_repos() + + for service in services: + process_service(args, service) + + +if __name__ == "__main__": + main() From 9efb301f51db4cb9661d5dd238c92bb31e3e3328 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Wed, 4 Mar 2026 11:16:24 +0000 Subject: [PATCH 4/7] Added better prompt (#80) Reviewed-on: https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-rework/pulls/80 Reviewed-by: Tino Schreiber Co-authored-by: Sebastian Gode Co-committed-by: Sebastian Gode --- tools/generate_meta.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/generate_meta.py b/tools/generate_meta.py index 71bbcff..cba62f8 100755 --- a/tools/generate_meta.py +++ b/tools/generate_meta.py @@ -87,6 +87,7 @@ def generate_description_with_llm(text, service_title, llm_api_url, model_name, content_preview = text[:2000].replace("\n", " ") prompt = ( f"Generate a meta description (40-160 chars) for: {service_title}." + f"This is a service from the cloud provider called 'T Cloud Public', do not mention other Cloud Providers or services from them." f"Content preview: {content_preview}." f"Output ONLY the description text, nothing else." ) @@ -160,6 +161,7 @@ def generate_keywords_with_llm(text, service_title, llm_api_url, model_name, api content_preview = text[:2000].replace("\n", " ") prompt = ( f"Generate up to 5 keywords (comma-separated) for: {service_title}. " + f"This is a service from the cloud provider called 'T Cloud Public', do not mention other Cloud Providers or services from them." f"Content preview: {content_preview}. " f"Output ONLY comma-separated keywords, nothing else." ) From e2acf0c60384e928ad4315b584f46f90a81c9c34 Mon Sep 17 00:00:00 2001 From: "Kvrivishvili, Iustina" Date: Wed, 25 Mar 2026 10:04:18 +0000 Subject: [PATCH 5/7] Add table suffix to cloud envs (#85) 1) Add table_suffix key to avoid EoD hardcoding Co-authored-by: Yusenka Reviewed-on: https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-rework/pulls/85 Reviewed-by: Hasko, Vladimir --- otc_metadata/data/cloud_environments/eu_de.yaml | 1 + otc_metadata/data/cloud_environments/swiss.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/otc_metadata/data/cloud_environments/eu_de.yaml b/otc_metadata/data/cloud_environments/eu_de.yaml index 7361ef4..c70efaa 100644 --- a/otc_metadata/data/cloud_environments/eu_de.yaml +++ b/otc_metadata/data/cloud_environments/eu_de.yaml @@ -1,5 +1,6 @@ --- name: eu_de +table_suffix: "" internal_org: docs public_org: opentelekomcloud-docs regions: diff --git a/otc_metadata/data/cloud_environments/swiss.yaml b/otc_metadata/data/cloud_environments/swiss.yaml index 8db37d8..41a8068 100644 --- a/otc_metadata/data/cloud_environments/swiss.yaml +++ b/otc_metadata/data/cloud_environments/swiss.yaml @@ -1,5 +1,6 @@ --- name: swiss +table_suffix: _swiss internal_org: docs-swiss public_org: opentelekomcloud-docs-swiss regions: From 1e4648067ddef83765afeec2a1f1f6debaa8d4e0 Mon Sep 17 00:00:00 2001 From: "Gode, Sebastian" Date: Tue, 31 Mar 2026 14:15:08 +0000 Subject: [PATCH 6/7] DMS RocketMQ public (#84) Reviewed-on: https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-rework/pulls/84 Reviewed-by: Hasko, Vladimir Co-authored-by: Gode, Sebastian Co-committed-by: Gode, Sebastian --- otc_metadata/data/documents/dms_rocketmq-api-ref.yaml | 2 +- otc_metadata/data/documents/dms_rocketmq-umn.yaml | 2 +- otc_metadata/data/services/dms_rocketmq.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/otc_metadata/data/documents/dms_rocketmq-api-ref.yaml b/otc_metadata/data/documents/dms_rocketmq-api-ref.yaml index da2c35b..e243dae 100644 --- a/otc_metadata/data/documents/dms_rocketmq-api-ref.yaml +++ b/otc_metadata/data/documents/dms_rocketmq-api-ref.yaml @@ -8,6 +8,6 @@ title: API Reference type: api-ref cloud_environments: - name: eu_de - visibility: internal + visibility: public pdf_visibility: public pdf_enabled: true diff --git a/otc_metadata/data/documents/dms_rocketmq-umn.yaml b/otc_metadata/data/documents/dms_rocketmq-umn.yaml index 4d78f0b..38f2a06 100644 --- a/otc_metadata/data/documents/dms_rocketmq-umn.yaml +++ b/otc_metadata/data/documents/dms_rocketmq-umn.yaml @@ -8,6 +8,6 @@ title: User Guide type: umn cloud_environments: - name: eu_de - visibility: internal + visibility: public pdf_visibility: public pdf_enabled: true diff --git a/otc_metadata/data/services/dms_rocketmq.yaml b/otc_metadata/data/services/dms_rocketmq.yaml index d4728c8..0ca1633 100644 --- a/otc_metadata/data/services/dms_rocketmq.yaml +++ b/otc_metadata/data/services/dms_rocketmq.yaml @@ -8,5 +8,5 @@ teams: permission: write cloud_environments: - name: eu_de - visibility: internal + visibility: public is_global: false From 12d16b7a2ddf1eeccf81e889719e737b9a9fdf04 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Wed, 1 Apr 2026 11:46:33 +0000 Subject: [PATCH 7/7] added preprod branch script (#87) Reviewed-on: https://gitea.eco.tsi-dev.otc-service.com/infra/otc-metadata-rework/pulls/87 Reviewed-by: Hasko, Vladimir Co-authored-by: Sebastian Gode Co-committed-by: Sebastian Gode --- tools/create_preprod_branch.py | 237 +++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100755 tools/create_preprod_branch.py diff --git a/tools/create_preprod_branch.py b/tools/create_preprod_branch.py new file mode 100755 index 0000000..c54e01f --- /dev/null +++ b/tools/create_preprod_branch.py @@ -0,0 +1,237 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import pathlib +import shutil + +from git import exc +from git import Repo + +import otc_metadata.services + +data = otc_metadata.services.Services() + + +def process_repositories(args, service): + """Create or update branch for a service repository""" + logging.debug(f"Processing service {service['service_title']}") + workdir = pathlib.Path(args.work_dir) + workdir.mkdir(exist_ok=True) + + repo_dir = None + repo_url = None + target_repo = None + error_list = [] + + for repo in service["repositories"]: + if repo["cloud_environments"][0] != args.cloud_environment: + continue + logging.debug(f"Processing repository {repo}") + repo_dir_path = pathlib.Path(workdir, repo["type"], repo["repo"]) + + if repo["environment"] == args.target_environment: + repo_dir = repo_dir_path + else: + logging.debug(f"Skipping repository {repo}") + continue + + repo_dir.mkdir(parents=True, exist_ok=True) + if repo["type"] == "gitea": + repo_url = ( + f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" + f"{repo['repo']}" + ) + elif repo["type"] == "github": + repo_url = f"git@github.com:/{repo['repo']}" + else: + logging.error(f"Repository type {repo['type']} is not supported") + error_list.append({ + "error": f"Repository type {repo['type']} is not supported", + "repo": repo['repo'] + }) + continue + + if repo_dir.exists(): + logging.debug(f"Repository {repo} already checked out") + try: + git_repo = Repo(repo_dir) + git_repo.remotes.origin.fetch() + except exc.InvalidGitRepositoryError: + logging.warning(f"Existing repository checkout is invalid, removing: {repo_dir}") + try: + shutil.rmtree(repo_dir) + except Exception as e: + logging.error(f"Error removing invalid repository {repo_dir}: {e}") + error_list.append({ + "error": f"Error removing invalid repository: {e}", + "repo": repo['repo'] + }) + continue + except Exception as e: + logging.error(f"Error fetching repository {repo_dir}: {e}") + error_list.append({ + "error": e, + "repo": repo['repo'] + }) + continue + + if not repo_dir.exists(): + try: + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") + except Exception as e: + logging.error(f"Error cloning repository {repo_url}: {e}") + error_list.append({ + "error": f"Error cloning repository {repo_url}: {e}", + "repo": repo['repo'] + }) + continue + + if repo["environment"] == args.target_environment: + target_repo = repo + break + + if not target_repo: + logging.info( + f"No repository for service {service['service_title']} " + f"for environment {args.target_environment} in cloud_environment {args.cloud_environment}" + ) + return + + branch_name = args.branch_name + + # Check if branch exists remotely + remote_branch_exists = False + try: + for ref in git_repo.remotes.origin.refs: + if ref.name == branch_name: + remote_branch_exists = True + break + # Also check via fetch if refs are not up to date + if not remote_branch_exists: + git_repo.remotes.origin.fetch() + for ref in git_repo.remotes.origin.refs: + if ref.name == branch_name: + remote_branch_exists = True + break + except Exception as e: + logging.warning(f"Error checking for remote branch: {e}") + + if remote_branch_exists: + # Branch exists, checkout and merge main into it + logging.debug(f"Branch {branch_name} exists, merging main into it") + try: + git_repo.heads[branch_name].checkout() + git_repo.git.merge("main", no_commit=True) + git_repo.index.commit(args.commit_description) + except Exception as e: + logging.error(f"Error merging main into branch {branch_name}: {e}") + error_list.append({ + "error": f"Error merging main into branch {branch_name}: {e}", + "repo": target_repo['repo'] + }) + return + else: + # Branch does not exist, create it from main + logging.debug(f"Branch {branch_name} does not exist, creating it") + try: + git_repo.heads.main.checkout() + new_branch = git_repo.create_head(branch_name) + new_branch.checkout() + except Exception as e: + logging.error(f"Error creating branch {branch_name}: {e}") + error_list.append({ + "error": f"Error creating branch {branch_name}: {e}", + "repo": target_repo['repo'] + }) + return + + # Push the branch + push_args = ["--set-upstream", "origin", branch_name] + if args.force_push: + push_args.append("--force") + try: + git_repo.git.push(*push_args) + logging.info(f"Successfully pushed branch {branch_name} for {target_repo['repo']}") + except Exception as e: + logging.error(f"Error pushing branch {branch_name}: {e}") + error_list.append({ + "error": f"Error pushing branch {branch_name}: {e}", + "repo": target_repo['repo'] + }) + + if len(error_list) != 0: + logging.error("The following errors have happened:") + logging.error(error_list) + + +def main(): + parser = argparse.ArgumentParser( + description="Create or update preprod branch for services." + ) + parser.add_argument( + "--target-environment", + required=True, + choices=["internal", "public"], + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", + ) + parser.add_argument( + "--branch-name", + default="preprod", + help="Branch name to be used for synchronizing.", + ) + parser.add_argument( + "--force-push", + action="store_true", + help="Whether to force push the commit" + ) + parser.add_argument( + "--commit-description", + default=( + "Sync with main\n\n" + "Performed-by: gitea/infra/otc-metadata/" + "tools/create_preprod_branch.py" + ), + help="Commit description for the commit", + ) + parser.add_argument( + "--cloud-environment", + required=True, + default="eu_de", + help="Cloud Environment. Default: eu_de", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) + services = [] + + if args.service_type: + services = [data.get_service_with_repo_by_service_type(service_type=args.service_type)] + else: + services = data.services_with_repos() + + for service in services: + process_repositories(args, service) + + +if __name__ == "__main__": + main()