295 lines
9.9 KiB
Python
295 lines
9.9 KiB
Python
#!/usr/bin/python
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import requests
|
|
import subprocess
|
|
import warnings
|
|
|
|
from git import exc
|
|
from git import Repo
|
|
from git import SymbolicReference
|
|
|
|
from ruamel.yaml import CommentedMap
|
|
from ruamel.yaml import YAML
|
|
|
|
import otc_metadata.services
|
|
|
|
data = otc_metadata.services.Services()
|
|
|
|
yaml = YAML()
|
|
|
|
api_session = requests.Session()
|
|
|
|
|
|
def load_zuul_config(workdir):
|
|
for path in [".zuul.yaml", "zuul.yaml"]:
|
|
test_path = pathlib.Path(workdir, path)
|
|
if test_path.exists():
|
|
with open(test_path, "r") as f:
|
|
zuul_config = yaml.load(f)
|
|
return (zuul_config, test_path)
|
|
|
|
|
|
def open_pr(args, repository, pr_data):
|
|
req = dict(
|
|
title=pr_data["title"],
|
|
body=pr_data["body"].replace("\\n", "\n"),
|
|
base=pr_data.get("base", "main"),
|
|
head=pr_data["head"],
|
|
)
|
|
if "assignees" in pr_data:
|
|
req["assignees"] = pr_data["assignees"]
|
|
if "labels" in pr_data:
|
|
req["labels"] = pr_data["labels"]
|
|
rsp = api_session.post(
|
|
f"{args.api_url}/repos/{repository}/pulls", json=req
|
|
)
|
|
if rsp.status_code != 201:
|
|
print(rsp.text)
|
|
print(f"Going to open PR with title {pr_data['title']} in {repository}")
|
|
|
|
|
|
def process_repositories(args, service):
|
|
"""Checkout repositories"""
|
|
logging.debug(f"Processing service {service}")
|
|
workdir = pathlib.Path(args.work_dir)
|
|
workdir.mkdir(exist_ok=True)
|
|
|
|
copy_to = None
|
|
repo_to = None
|
|
|
|
for repo in service["repositories"]:
|
|
logging.debug(f"Processing repository {repo}")
|
|
repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"])
|
|
|
|
if repo["environment"] != args.environment:
|
|
continue
|
|
|
|
repo_dir.mkdir(parents=True, exist_ok=True)
|
|
if repo["type"] == "gitea":
|
|
repo_url = (
|
|
f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/"
|
|
f"{repo['repo']}"
|
|
)
|
|
elif repo["type"] == "github":
|
|
repo_url = f"git@github.com:/{repo['repo']}"
|
|
else:
|
|
logging.error(f"Repository type {repo['type']} is not supported")
|
|
exit(1)
|
|
|
|
if repo_dir.exists():
|
|
logging.debug(f"Repository {repo} already checked out")
|
|
try:
|
|
git_repo = Repo(repo_dir)
|
|
git_repo.remotes.origin.update()
|
|
git_repo.remotes.origin.fetch()
|
|
git_repo.heads.main.checkout()
|
|
git_repo.remotes.origin.pull()
|
|
except exc.InvalidGitRepositoryError:
|
|
logging.error("Existing repository checkout is bad")
|
|
repo_dir.rmdir()
|
|
|
|
if not repo_dir.exists():
|
|
try:
|
|
git_repo = Repo.clone_from(repo_url, repo_dir, branch="main")
|
|
except Exception:
|
|
logging.error(f"Error cloning repository {repo_url}")
|
|
return
|
|
|
|
branch_name = f"{args.branch_name}"
|
|
if args.branch_force:
|
|
logging.debug("Dropping current branch")
|
|
try:
|
|
git_repo.delete_head(branch_name, force=True)
|
|
except exc.GitCommandError:
|
|
pass
|
|
try:
|
|
new_branch = git_repo.create_head(branch_name, "main")
|
|
except Exception as ex:
|
|
logging.warning(f"Skipping service {service} due to {ex}")
|
|
return
|
|
new_branch.checkout()
|
|
|
|
(zuul_config, zuul_file_name) = load_zuul_config(repo_dir)
|
|
zuul_templates = None
|
|
zuul_jobs = dict()
|
|
zuul_new_jobs = list()
|
|
zuul_config_updated = False
|
|
for item in zuul_config:
|
|
if "project" in item.keys():
|
|
project = item["project"]
|
|
zuul_templates = project.setdefault("templates", [])
|
|
if not zuul_templates:
|
|
zuul_templates = []
|
|
elif "job" in item.keys():
|
|
job = item["job"]
|
|
zuul_jobs[job["name"]] = job
|
|
print(f"Existing jobs {zuul_jobs}")
|
|
if "helpcenter-base-jobs" not in zuul_templates:
|
|
zuul_templates.append("helpcenter-base-jobs")
|
|
zuul_config_updated = True
|
|
|
|
job_suffix = (
|
|
"-hc-int-jobs" if args.environment == "internal" else "-hc-jobs"
|
|
)
|
|
for doc in data.docs_by_service_type(service["service_type"]):
|
|
logging.debug(f"Analyzing document {doc}")
|
|
if not doc.get("type"):
|
|
continue
|
|
if doc["type"] == "dev":
|
|
doc_type = "dev-guide"
|
|
else:
|
|
doc_type = doc["type"]
|
|
template_name = f"{doc_type}{job_suffix}"
|
|
if doc_type in ["api-ref", "umn", "dev-guide"]:
|
|
if template_name not in zuul_templates:
|
|
zuul_templates.append(template_name)
|
|
else:
|
|
job_name = f"build-otc-{doc['service_type']}-{doc_type}"
|
|
if job_name not in zuul_jobs:
|
|
zuul_config_updated = True
|
|
zuul_new_jobs.append(
|
|
dict(
|
|
job=dict(
|
|
name=job_name,
|
|
parent="otc-tox-docs",
|
|
description=(
|
|
f"Build {doc_type} document using tox"
|
|
),
|
|
files=[
|
|
f"^{doc['rst_location']}/.*"
|
|
],
|
|
vars=dict(
|
|
tox_envlist=doc_type
|
|
)
|
|
)
|
|
)
|
|
)
|
|
|
|
if zuul_config_updated:
|
|
for new_job in zuul_new_jobs:
|
|
zuul_config.insert(0, new_job)
|
|
|
|
for item in zuul_config:
|
|
if "project" in item.keys():
|
|
project = item["project"]
|
|
project["templates"] = zuul_templates
|
|
# Ensure new jobs are in check
|
|
if len(zuul_new_jobs) > 0:
|
|
project.setdefault(
|
|
"check",
|
|
CommentedMap(jobs=[])
|
|
)
|
|
project["check"].yaml_set_comment_before_after_key(
|
|
key="jobs",
|
|
indent=6,
|
|
before=(
|
|
"Separate documents are rendered in check, "
|
|
"while published through regular "
|
|
"otc-tox-docs job part of the basic template"
|
|
)
|
|
)
|
|
project["check"]["jobs"].extend(
|
|
[x["job"]["name"] for x in zuul_new_jobs])
|
|
|
|
|
|
#yaml.indent(offset=2, sequence=2)
|
|
with open(zuul_file_name, "w") as f:
|
|
yaml.dump(zuul_config, f)
|
|
git_repo.index.add([zuul_file_name.name])
|
|
if len(git_repo.index.diff("HEAD")) == 0:
|
|
# Nothing to commit
|
|
logging.debug(
|
|
"No changes for service %s required" %
|
|
service["service_type"])
|
|
return
|
|
|
|
git_repo.index.commit(
|
|
(
|
|
"Update zuul.yaml file\n\n"
|
|
"Performed-by: gitea/infra/otc-metadata/tools/update_zuul_project_config.py"
|
|
)
|
|
)
|
|
push_args = ["--set-upstream", "origin", branch_name]
|
|
if args.branch_force:
|
|
push_args.append("--force")
|
|
|
|
git_repo.git.push(*push_args)
|
|
if repo["type"] == "github":
|
|
subprocess.run(
|
|
args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True
|
|
)
|
|
elif repo["type"] == "gitea":
|
|
open_pr(
|
|
args,
|
|
repo["repo"],
|
|
dict(
|
|
title="Update zuul config",
|
|
body="Update zuul config templates",
|
|
head=branch_name,
|
|
),
|
|
)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Update zuul.yaml file in repositories."
|
|
)
|
|
parser.add_argument(
|
|
"--environment",
|
|
required=True,
|
|
help="Repository Environment",
|
|
)
|
|
parser.add_argument("--service-type", help="Service to update")
|
|
parser.add_argument(
|
|
"--work-dir",
|
|
required=True,
|
|
help="Working directory to use for repository checkout.",
|
|
)
|
|
parser.add_argument(
|
|
"--branch-name",
|
|
default="zuul",
|
|
help="Branch name to be used for synchronizing.",
|
|
)
|
|
parser.add_argument(
|
|
"--branch-force",
|
|
action="store_true",
|
|
help="Whether to force branch recreation.",
|
|
)
|
|
parser.add_argument("--token", metavar="token", help="API token")
|
|
parser.add_argument("--api-url", help="API base url of the Git hoster")
|
|
|
|
args = parser.parse_args()
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
services = []
|
|
if args.service_type:
|
|
services = [data.service_dict.get(args.service_type)]
|
|
else:
|
|
services = data.all_services
|
|
|
|
if args.token:
|
|
api_session.headers.update({"Authorization": f"token {args.token}"})
|
|
|
|
for service in services:
|
|
process_repositories(args, service)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|