From ba65aca9eb6618a42ae51f0ec5aff110bf244c96 Mon Sep 17 00:00:00 2001 From: gtema Date: Mon, 2 Jan 2023 18:56:58 +0000 Subject: [PATCH] Update 'zuul.yaml' Reviewed-by: Goncharov, Artem Co-authored-by: gtema Co-committed-by: gtema --- .stestr.conf | 2 +- otc_metadata/data/__init__.py | 1 + otc_metadata/docs.py | 2 +- otc_metadata/services.py | 8 +- otc_metadata/tests/base.py | 23 ---- ...t_otc-metadata.py => test_otc_metadata.py} | 4 +- setup.cfg | 6 +- setup.py | 1 - test-requirements.txt | 9 +- tools/bootstrap_repositories.py | 90 +++++++-------- tools/convert_data.py | 13 +-- tools/generate_doc_confpy.py | 109 ++++++++---------- tools/generate_doc_gitcontrol_repos.py | 75 ++++++------ tools/generate_docexports_data.py | 4 +- tools/open_doc_issue.py | 56 +++++---- tools/sort_metadata.py | 4 +- tools/sync_doc_repo.py | 83 +++++++------ tools/update_zuul_project_configs.py | 14 +-- tox.ini | 34 +----- zuul.yaml | 12 +- 20 files changed, 240 insertions(+), 310 deletions(-) delete mode 100644 otc_metadata/tests/base.py rename otc_metadata/tests/{test_otc-metadata.py => test_otc_metadata.py} (90%) diff --git a/.stestr.conf b/.stestr.conf index c025c8e..6b9459b 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,3 +1,3 @@ [DEFAULT] -test_path=./otc-metadata/tests +test_path=./otc_metadata/tests/ top_dir=./ diff --git a/otc_metadata/data/__init__.py b/otc_metadata/data/__init__.py index fc5139d..2102a7e 100644 --- a/otc_metadata/data/__init__.py +++ b/otc_metadata/data/__init__.py @@ -28,6 +28,7 @@ def read_data(filename): with open(filepath, 'r') as fd: return yaml.safe_load(fd) + def rewrite_data(filename, data): """Rewrites data formatting it diff --git a/otc_metadata/docs.py b/otc_metadata/docs.py index 781cc9f..733add1 100644 --- a/otc_metadata/docs.py +++ b/otc_metadata/docs.py @@ -15,7 +15,7 @@ import copy import otc_metadata.data -__all__ = ['Docs'] +__all__ = ['Service'] BUILTIN_DATA = otc_metadata.data.read_data('docs.yaml') diff --git a/otc_metadata/services.py b/otc_metadata/services.py index fbb7c6f..8fdc170 100644 --- a/otc_metadata/services.py +++ b/otc_metadata/services.py @@ -40,7 +40,8 @@ class Services(object): # sort docs list by _ self._service_data["documents"] = sorted( - sorted_docs, key=lambda x: f"{x.get('service_type')}{x.get('title')}" + sorted_docs, + key=lambda x: f"{x.get('service_type')}{x.get('title')}" ) # sort services by <service_type>_<service_title> self._service_data["services"] = sorted( @@ -183,7 +184,10 @@ class Services(object): ] if "repositories" in srv and environment: for repo in srv["repositories"]: - if "environment" in repo and repo["environment"] == environment: + if ( + "environment" in repo + and repo["environment"] == environment + ): srv_res["repository"] = repo["repo"] for doc in self.all_docs: if ( diff --git a/otc_metadata/tests/base.py b/otc_metadata/tests/base.py deleted file mode 100644 index 8a06ce9..0000000 --- a/otc_metadata/tests/base.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2010-2011 OpenStack Foundation -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from unittest import TestCase - - -class TestCase(TestCase): - - """Test case base class for all unit tests.""" diff --git a/otc_metadata/tests/test_otc-metadata.py b/otc_metadata/tests/test_otc_metadata.py similarity index 90% rename from otc_metadata/tests/test_otc-metadata.py rename to otc_metadata/tests/test_otc_metadata.py index 24401b2..eb22b69 100644 --- a/otc_metadata/tests/test_otc-metadata.py +++ b/otc_metadata/tests/test_otc_metadata.py @@ -19,10 +19,10 @@ test_otc-metadata Tests for `otc-metadata` module. """ -from otc-metadata.tests import base +from unittest import TestCase -class TestOtc-metadata(base.TestCase): +class TestOtcMetadata(TestCase): def test_something(self): pass diff --git a/setup.cfg b/setup.cfg index ab7f3a0..ad735b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,11 @@ [metadata] name = otc-metadata summary = Metadata about OTC for Ecosystem -description-file = +description_file = README.rst author = Open Telekom Cloud -home-page = https://open.telekom.cloud/ -python-requires = >=3.6 +home_page = https://open.telekom.cloud/ +python_requires = >=3.6 classifier = Environment :: OpenStack Intended Audience :: Information Technology diff --git a/setup.py b/setup.py index 6016f37..7f1bbe8 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools setuptools.setup( diff --git a/test-requirements.txt b/test-requirements.txt index 7f21c17..8a2bd6e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,9 +2,6 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=3.0,<3.1 # Apache-2.0 - -coverage>=4.0,!=4.4 # Apache-2.0 -python-subunit>=0.0.18 # Apache-2.0/BSD -stestr>=1.0.0 # Apache-2.0 -testtools>=1.4.0 # MIT +stestr>=2.0.0 # Apache-2.0 +testtools>=2.2.0 # MIT +flake8 diff --git a/tools/bootstrap_repositories.py b/tools/bootstrap_repositories.py index 85f6d08..719ad38 100644 --- a/tools/bootstrap_repositories.py +++ b/tools/bootstrap_repositories.py @@ -15,18 +15,18 @@ import argparse import logging -import os +# import os import pathlib import subprocess -import warnings +# import warnings -from git import exc +# from git import exc from git import Repo -from git import SymbolicReference +# from git import SymbolicReference -from jinja2 import PackageLoader -from jinja2 import Environment -from jinja2 import select_autoescape +# from jinja2 import PackageLoader +# from jinja2 import Environment +# from jinja2 import select_autoescape from cookiecutter.main import cookiecutter @@ -36,7 +36,7 @@ data = otc_metadata.services.Services() def create_repo(repo, repo_dir, service): - repo_owner, repo_name = repo['repo'].split('/') + repo_owner, repo_name = repo["repo"].split("/") git_repo = None if repo_dir.exists(): @@ -44,20 +44,18 @@ def create_repo(repo, repo_dir, service): return logging.info(f"Creating repository {repo_owner}/{repo_name}") - if repo['type'] == 'gitea': + if repo["type"] == "gitea": repo_url = ( f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" f"{repo['repo']}" ) git_fqdn = "gitea.eco.tsi-dev.otc-service.com" - elif repo['type'] == 'github': + elif repo["type"] == "github": repo_url = f"git@github.com:/{repo['repo']}" git_fqdn = "github.com" try: - git_repo = Repo.clone_from( - repo_url, - repo_dir, branch='main') + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") except Exception: logging.debug("Error") pass @@ -67,17 +65,26 @@ def create_repo(repo, repo_dir, service): if repo["type"] == "gitea": subprocess.run( - args=["tea", "repo", "create", - "--name", repo_name, - "--owner", repo_owner, - ] - ) + args=[ + "tea", + "repo", + "create", + "--name", + repo_name, + "--owner", + repo_owner, + ] + ) elif repo["type"] == "github": subprocess.run( - args=["gh", "repo", "create", - repo["repo"], "--public", - ] - ) + args=[ + "gh", + "repo", + "create", + repo["repo"], + "--public", + ] + ) cookiecutter( template="https://github.com/opentelekomcloud/docs-cookiecutter", output_dir=repo_dir.parent, @@ -88,29 +95,26 @@ def create_repo(repo, repo_dir, service): repo_name=repo_name, project_short_description=( f"Documentation project for {service['service_title']} " - "service"), - overwrite_if_exists=True - ) + "service" + ), + overwrite_if_exists=True, + ), ) git_repo = Repo(repo_dir) - git_repo.create_remote('origin', repo_url) + git_repo.create_remote("origin", repo_url) git_repo.remotes.origin.fetch() - git_repo.git.push('--set-upstream', 'origin', 'main') + git_repo.git.push("--set-upstream", "origin", "main") def process_repositories(args, service): - """Checkout repositories - - """ + """Checkout repositories""" logging.debug(f"Processing service {service}") workdir = pathlib.Path(args.work_dir) workdir.mkdir(exist_ok=True) for repo in service["repositories"]: logging.debug(f"Processing repository {repo}") - repo_dir = pathlib.Path( - workdir, repo["type"], repo["repo"] - ) + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) if repo["environment"] != args.target_environment: continue @@ -118,26 +122,22 @@ def process_repositories(args, service): checkout_exists = repo_dir.exists() logging.debug(f"Repository {repo} exists {checkout_exists}") repo_dir.parent.mkdir(parents=True, exist_ok=True) - if True: # not checkout_exists: + if True: # not checkout_exists: create_repo(repo, repo_dir, service) def main(): - parser = argparse.ArgumentParser( - description='Bootstrap repositories.') + parser = argparse.ArgumentParser(description="Bootstrap repositories.") parser.add_argument( - '--target-environment', + "--target-environment", required=True, - help='Environment to be used as a source' + help="Environment to be used as a source", ) + parser.add_argument("--service-type", help="Service to update") parser.add_argument( - '--service-type', - help='Service to update' - ) - parser.add_argument( - '--work-dir', + "--work-dir", required=True, - help='Working directory to use for repository checkout.' + help="Working directory to use for repository checkout.", ) args = parser.parse_args() @@ -152,5 +152,5 @@ def main(): process_repositories(args, service) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/convert_data.py b/tools/convert_data.py index 2154974..1d7fb3e 100644 --- a/tools/convert_data.py +++ b/tools/convert_data.py @@ -1,4 +1,3 @@ -import copy import re import otc_metadata.services @@ -7,21 +6,17 @@ from ruamel.yaml import YAML data = otc_metadata.services.Services() new_data = data._service_data -#services = data.service_dict +# services = data.service_dict for doc in new_data["documents"]: hc_location = None link = doc.get("link") if link: print(f"Parsing {link}") - #(p1, p2) = link.split("/") - doc["link"] = re.sub( - r"/(.*)/(.*)/", - r"/\2/\1/", - link - ) + # (p1, p2) = link.split("/") + doc["link"] = re.sub(r"/(.*)/(.*)/", r"/\2/\1/", link) _yaml = YAML() _yaml.indent(mapping=2, sequence=4, offset=2) -with open('new.yaml', 'w') as fd: +with open("new.yaml", "w") as fd: _yaml.dump(new_data, fd) diff --git a/tools/generate_doc_confpy.py b/tools/generate_doc_confpy.py index a86f36c..511e475 100644 --- a/tools/generate_doc_confpy.py +++ b/tools/generate_doc_confpy.py @@ -21,7 +21,6 @@ import subprocess from git import exc from git import Repo -from git import SymbolicReference from jinja2 import PackageLoader from jinja2 import Environment @@ -35,9 +34,7 @@ api_session = requests.Session() def process_repositories(args, service): - """Checkout repositories - - """ + """Checkout repositories""" logging.debug(f"Processing service {service}") workdir = pathlib.Path(args.work_dir) workdir.mkdir(exist_ok=True) @@ -49,17 +46,15 @@ def process_repositories(args, service): git_fqdn = None env = Environment( - loader=PackageLoader("otc_metadata"), - autoescape=select_autoescape()) + loader=PackageLoader("otc_metadata"), autoescape=select_autoescape() + ) conf_py_template = env.get_template("conf.py.j2") tox_ini_template = env.get_template("tox.ini.j2") doc_requirements_template = env.get_template("doc_requirements.txt.j2") for repo in service["repositories"]: logging.debug(f"Processing repository {repo}") - repo_dir = pathlib.Path( - workdir, repo["type"], repo["repo"] - ) + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) if repo["environment"] == args.target_environment: copy_to = repo_dir @@ -68,13 +63,13 @@ def process_repositories(args, service): continue repo_dir.mkdir(parents=True, exist_ok=True) - if repo["type"] == 'gitea': + if repo["type"] == "gitea": repo_url = ( f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" f"{repo['repo']}" ) git_fqdn = "gitea.eco.tsi-dev.otc-service.com" - elif repo["type"] == 'github': + elif repo["type"] == "github": repo_url = f"git@github.com:/{repo['repo']}" else: logging.error(f"Repository type {repo['type']} is not supported") @@ -93,9 +88,7 @@ def process_repositories(args, service): if not repo_dir.exists(): try: - git_repo = Repo.clone_from( - repo_url, - repo_dir, branch='main') + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") except Exception: logging.error(f"Error cloning repository {repo_url}") return @@ -109,7 +102,8 @@ def process_repositories(args, service): if not target_repo: logging.info( f"No repository service {service['service_title']}" - f"for environment {args.target_environment}") + f"for environment {args.target_environment}" + ) return branch_name = f"{args.branch_name}" @@ -122,7 +116,7 @@ def process_repositories(args, service): print(e) pass try: - new_branch = repo_to.create_head(branch_name, 'main') + new_branch = repo_to.create_head(branch_name, "main") except Exception as e: logging.warning(f"Skipping service {service} due to {e}") return @@ -133,7 +127,7 @@ def process_repositories(args, service): for doc in service_docs: logging.debug(f"Analyzing document {doc}") - conf_py_path = pathlib.Path(copy_to, doc['rst_location'], 'conf.py') + conf_py_path = pathlib.Path(copy_to, doc["rst_location"], "conf.py") if not conf_py_path.exists(): logging.info(f"Path for document {doc['title']} does not exist") conf_py_path.parent.mkdir(parents=True, exist_ok=True) @@ -141,7 +135,7 @@ def process_repositories(args, service): repo_name=target_repo["repo"], project=service["service_title"], # pdf_name=doc["pdf_name"], - title=f"{service['service_title']} - {doc['title']}" + title=f"{service['service_title']} - {doc['title']}", ) if "pdf_name" in doc: context["pdf_name"] = doc["pdf_name"] @@ -156,57 +150,60 @@ def process_repositories(args, service): logo_url="https://docs-int.otc-service.com", ) conf_py_content = conf_py_template.render(**context) - with open(conf_py_path, 'w', encoding='utf-8', newline='') as out: + with open(conf_py_path, "w", encoding="utf-8", newline="") as out: logging.debug(f"Generating {conf_py_path} from template...") out.write(conf_py_content) repo_to.index.add([doc["rst_location"]]) if args.update_tox: - context = dict( - docs=[] - ) + context = dict(docs=[]) for doc in service_docs: if doc["type"] == "dev": doc["type"] = "dev-guide" context["docs"].append(doc) tox_ini_content = tox_ini_template.render(**context) - tox_ini_path = pathlib.Path(copy_to, 'tox.ini') + tox_ini_path = pathlib.Path(copy_to, "tox.ini") doc_requirements_content = doc_requirements_template.render(**context) - doc_requirements_path = pathlib.Path(copy_to, "doc", 'requirements.txt') + doc_requirements_path = pathlib.Path( + copy_to, "doc", "requirements.txt" + ) doc_requirements_path.parent.mkdir(parents=True, exist_ok=True) - with open(tox_ini_path, 'w', encoding='utf-8', newline='') as out: + with open(tox_ini_path, "w", encoding="utf-8", newline="") as out: logging.debug(f"Generating {tox_ini_path} from template...") out.write(tox_ini_content) repo_to.index.add(["tox.ini"]) - with open(doc_requirements_path, 'w', encoding='utf-8', newline='') as out: - logging.debug(f"Generating {doc_requirements_path} from template...") + with open( + doc_requirements_path, "w", encoding="utf-8", newline="" + ) as out: + logging.debug( + f"Generating {doc_requirements_path} from template..." + ) out.write(doc_requirements_content) repo_to.index.add(["doc/requirements.txt"]) - if len(repo_to.index.diff("HEAD")) == 0: # Nothing to commit logging.debug( - "No changes for service %s required" % service["service_type"]) + "No changes for service %s required" % service["service_type"] + ) return repo_to.index.commit( ( - f"Update tox.ini && conf.py file\n\n" - "Performed-by: gitea/infra/otc-metadata/tools/generate_doc_confpy.py" + "Update tox.ini && conf.py file\n\n" + "Performed-by: gitea/infra/otc-metadata/" + "tools/generate_doc_confpy.py" ) ) push_args = ["--set-upstream", "origin", branch_name] if args.branch_force: push_args.append("--force") repo_to.git.push(*push_args) - if 'github' in url_to: + if "github" in url_to: subprocess.run( - args=['gh', 'pr', 'create', '-f'], - cwd=copy_to, - check=True + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True ) - elif 'gitea' in url_to and args.token: + elif "gitea" in url_to and args.token: open_pr( args, repo["repo"], @@ -235,40 +232,36 @@ def open_pr(args, repository, pr_data): ) if rsp.status_code != 201: print(rsp.text) - #print(f"Going to open PR with title {pr_data['title']} in {repository}") + # print(f"Going to open PR with title {pr_data['title']} in {repository}") def main(): parser = argparse.ArgumentParser( - description='Update conf.py file in repositories.') + description="Update conf.py file in repositories." + ) parser.add_argument( - '--target-environment', + "--target-environment", required=True, - help='Environment to be used as a source' + help="Environment to be used as a source", + ) + parser.add_argument("--service-type", help="Service to update") + parser.add_argument( + "--update-tox", action="store_true", help="Whether to update tox.ini." ) parser.add_argument( - '--service-type', - help='Service to update' - ) - parser.add_argument( - '--update-tox', - action='store_true', - help='Whether to update tox.ini.' - ) - parser.add_argument( - '--work-dir', + "--work-dir", required=True, - help='Working directory to use for repository checkout.' + help="Working directory to use for repository checkout.", ) parser.add_argument( - '--branch-name', - default='confpy', - help='Branch name to be used for synchronizing.' + "--branch-name", + default="confpy", + help="Branch name to be used for synchronizing.", ) parser.add_argument( - '--branch-force', - action='store_true', - help='Whether to force branch recreation.' + "--branch-force", + action="store_true", + help="Whether to force branch recreation.", ) parser.add_argument("--token", metavar="token", help="API token") parser.add_argument("--api-url", help="API base url of the Git hoster") @@ -288,5 +281,5 @@ def main(): process_repositories(args, service) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/generate_doc_gitcontrol_repos.py b/tools/generate_doc_gitcontrol_repos.py index 2aff192..16bdfe6 100644 --- a/tools/generate_doc_gitcontrol_repos.py +++ b/tools/generate_doc_gitcontrol_repos.py @@ -24,9 +24,7 @@ data = otc_metadata.services.Services() def process_services(args, services): - """Process services - - """ + """Process services""" gitea_bp = dict( branch_name="main", enable_approvals_whitelist=True, @@ -36,7 +34,7 @@ def process_services(args, services): enable_push=False, status_check_contexts=["gl/check"], enable_merge_whitelist=True, - merge_whitelist_usernames=["zuul"] + merge_whitelist_usernames=["zuul"], ) gitea_repo_template = dict( default_branch="main", @@ -51,7 +49,7 @@ def process_services(args, services): allow_squash_merge=False, allow_rebase_merge=False, default_merge_style="squash", - branch_protections=[] + branch_protections=[], ) github_bp = dict( branch_name="main", @@ -62,7 +60,7 @@ def process_services(args, services): enable_push=False, status_check_contexts=["gl/check"], enable_merge_whitelist=True, - merge_whitelist_usernames=["zuul"] + merge_whitelist_usernames=["zuul"], ) github_repo_template = dict( default_branch="main", @@ -77,7 +75,7 @@ def process_services(args, services): allow_squash_merge=True, allow_rebase_merge=False, allow_update_branch=True, - branch_protections=[] + branch_protections=[], ) for service in services: @@ -90,7 +88,7 @@ def process_services(args, services): if repo["environment"] != args.target_environment: continue - if repo["type"] == 'gitea': + if repo["type"] == "gitea": teams = [] branch_protections_main = copy.deepcopy(gitea_bp) if "teams" in repo: @@ -99,8 +97,9 @@ def process_services(args, services): teams_def = service["teams"] if teams_def: for team in teams_def: - branch_protections_main["approvals_whitelist_teams"].append( - team["name"]) + branch_protections_main[ + "approvals_whitelist_teams" + ].append(team["name"]) teams.append(team["name"]) data = copy.deepcopy(gitea_repo_template) data["description"] = ( @@ -109,10 +108,10 @@ def process_services(args, services): ) data["branch_protections"].append(branch_protections_main) data["teams"] = teams - repo_name = repo["repo"].split('/')[1] + repo_name = repo["repo"].split("/")[1] config = {repo_name: data} - elif repo["type"] == 'github': + elif repo["type"] == "github": teams = [] branch_protections_main = copy.deepcopy(github_bp) if "teams" in repo: @@ -121,50 +120,58 @@ def process_services(args, services): teams_def = service["teams"] if teams_def: for team in teams_def: - teams.append({ - "slug": team["name"], - "permission": "push" if team["permission"] == "write" else "pull" - }) + teams.append( + { + "slug": team["name"], + "permission": "push" + if team["permission"] == "write" + else "pull", + } + ) data = copy.deepcopy(github_repo_template) data["description"] = ( - f"Open Telekom Cloud {service['service_title']} " - f"docs" + f"Open Telekom Cloud {service['service_title']} " f"docs" + ) + data["branch_protections"].append( + {"branch": "main", "template": "zuul"} ) - data["branch_protections"].append({"branch": "main", - "template": "zuul"}) data["teams"] = teams - repo_name = repo["repo"].split('/')[1] + repo_name = repo["repo"].split("/")[1] config = {repo_name: data} else: logging.error( - "Repository type %s is not supported", - repo['type']) + "Repository type %s is not supported", repo["type"] + ) exit(1) if data: with open( - f"{args.work_dir}/{repo_name}.yml", 'w', - encoding='utf-8', newline='' + f"{args.work_dir}/{repo_name}.yml", + "w", + encoding="utf-8", + newline="", ) as out: logging.debug( "Generating %s config from template...", - service['service_type']) + service["service_type"], + ) yaml.dump(config, out) def main(): parser = argparse.ArgumentParser( - description='Update conf.py file in repositories.') - parser.add_argument( - '--target-environment', - required=True, - help='Environment to be used as a source' + description="Update conf.py file in repositories." ) parser.add_argument( - '--work-dir', + "--target-environment", required=True, - help='Working directory to use for repository checkout.' + help="Environment to be used as a source", + ) + parser.add_argument( + "--work-dir", + required=True, + help="Working directory to use for repository checkout.", ) args = parser.parse_args() @@ -173,5 +180,5 @@ def main(): process_services(args, data.all_services) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/generate_docexports_data.py b/tools/generate_docexports_data.py index 4c76aaf..8b8012a 100644 --- a/tools/generate_docexports_data.py +++ b/tools/generate_docexports_data.py @@ -13,7 +13,9 @@ def main(): _yaml = YAML() _yaml.indent(mapping=2, sequence=4, offset=2) - sys.stdout.write("# Auto-generated by otc_metadata.generate_docexports.data\n") + sys.stdout.write( + "# Auto-generated by otc_metadata.generate_docexports.data\n" + ) _yaml.dump(data.docs_html_by_category("internal"), sys.stdout) diff --git a/tools/open_doc_issue.py b/tools/open_doc_issue.py index f7d49a9..bf4b34b 100644 --- a/tools/open_doc_issue.py +++ b/tools/open_doc_issue.py @@ -1,9 +1,9 @@ #!/usr/bin/env python import argparse -import re +# import re import requests -import sys +# import sys import otc_metadata.services @@ -12,8 +12,7 @@ api_session = requests.Session() def open_issue(args, repository, issue_data): req = dict( - title=issue_data["title"], - body=issue_data["body"].replace("\\n", "\n") + title=issue_data["title"], body=issue_data["body"].replace("\\n", "\n") ) if "assignees" in issue_data: req["assignees"] = issue_data["assignees"] @@ -21,57 +20,54 @@ def open_issue(args, repository, issue_data): req["labels"] = issue_data["labels"] print(req) rsp = api_session.post( - f"{args.api_url}/repos/{repository}/issues", - json=req + f"{args.api_url}/repos/{repository}/issues", json=req ) if rsp.status_code != 201: print(rsp.text) - print(f"Going to open issue with title {issue_data['title']} in {repository}") + print( + f"Going to open issue with title {issue_data['title']} in {repository}" + ) def main(): - parser = argparse.ArgumentParser(description='Open Issue for every document.') + parser = argparse.ArgumentParser( + description="Open Issue for every document." + ) + parser.add_argument("token", metavar="token", help="API token") + parser.add_argument("--api-url", help="API base url of the Git hoster") + parser.add_argument("--environment", help="Environment for the repository") + parser.add_argument("--title", required=True, help="Issue title") + parser.add_argument("--body", required=True, help="Issue body") parser.add_argument( - 'token', metavar='token', help='API token') - parser.add_argument( - '--api-url', help='API base url of the Git hoster' + "--repo", + help="Repository to report issue in (instead of doc repository).", ) parser.add_argument( - '--environment', help='Environment for the repository' + "--assignee", + help="Issue assignee to use instead of document service assignees.", ) parser.add_argument( - '--title', required=True, help='Issue title' - ) - parser.add_argument( - '--body', required=True, help='Issue body' - ) - parser.add_argument( - '--repo', help='Repository to report issue in (instead of doc repository).' - ) - parser.add_argument( - '--assignee', help='Issue assignee to use instead of document service assignees.' - ) - parser.add_argument( - '--labels', help='Issue labels to use (comma separated list of label IDs).' + "--labels", + help="Issue labels to use (comma separated list of label IDs).", ) args = parser.parse_args() data = otc_metadata.services.Services() - api_session.headers.update({'Authorization': f"token {args.token}"}) + api_session.headers.update({"Authorization": f"token {args.token}"}) for doc in data.all_docs_full(environment=args.environment): - issue_data=dict( + issue_data = dict( title=args.title.format(**doc), body=args.body.format(**doc), - repository=doc["repository"] + repository=doc["repository"], ) if "assignees" in doc: issue_data["assignees"] = doc["assignees"] if args.assignee: issue_data["assignees"] = [args.assignee] if args.labels: - issue_data["labels"] = [int(x) for x in args.labels.split(',')] + issue_data["labels"] = [int(x) for x in args.labels.split(",")] open_issue(args, args.repo or doc["repository"], issue_data) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/sort_metadata.py b/tools/sort_metadata.py index d9a8470..caeae59 100644 --- a/tools/sort_metadata.py +++ b/tools/sort_metadata.py @@ -1,7 +1,7 @@ -import copy +# import copy import otc_metadata.services -from ruamel.yaml import YAML +# from ruamel.yaml import YAML data = otc_metadata.services.Services() diff --git a/tools/sync_doc_repo.py b/tools/sync_doc_repo.py index cc1e5d4..abdfc79 100644 --- a/tools/sync_doc_repo.py +++ b/tools/sync_doc_repo.py @@ -32,9 +32,7 @@ data = otc_metadata.services.Services() def process_repositories(args, service): - """Checkout repositories - - """ + """Checkout repositories""" workdir = pathlib.Path(args.work_dir) workdir.mkdir(exist_ok=True) @@ -46,9 +44,7 @@ def process_repositories(args, service): for repo in service["repositories"]: logging.debug(f"Processing repository {repo}") - repo_dir = pathlib.Path( - workdir, repo["type"], repo["repo"] - ) + repo_dir = pathlib.Path(workdir, repo["type"], repo["repo"]) if repo["environment"] == args.source_environment: copy_from = repo_dir @@ -59,22 +55,20 @@ def process_repositories(args, service): checkout_exists = repo_dir.exists() repo_dir.mkdir(parents=True, exist_ok=True) - if repo["type"] == 'gitea': + if repo["type"] == "gitea": repo_url = ( f"ssh://git@gitea.eco.tsi-dev.otc-service.com:2222/" f"{repo['repo']}" ) - elif repo["type"] == 'github': + elif repo["type"] == "github": repo_url = f"git@github.com:/{repo['repo']}" else: logging.error(f"Repository type {repo['type']} is not supported") exit(1) if not checkout_exists: - git_repo = Repo.clone_from( - repo_url, - repo_dir, branch='main') + git_repo = Repo.clone_from(repo_url, repo_dir, branch="main") else: - logging.debug(f"Checkout already exists") + logging.debug("Checkout already exists") git_repo = Repo(repo_dir) git_repo.remotes.origin.fetch() git_repo.heads.main.checkout() @@ -90,25 +84,29 @@ def process_repositories(args, service): if not copy_from or not copy_to: logging.warn( "Not synchronizing documents of the service. " - "Target or source not known.") + "Target or source not known." + ) for doc in data.docs_by_service_type(service["service_type"]): logging.debug(f"Analyzing document {doc}") if args.document_type and doc.get("type") != args.document_type: logging.info( f"Skipping synchronizing {doc['title']} " - f"due to the doc-type filter.") + f"due to the doc-type filter." + ) continue if doc.get("environment"): logging.info( f"Skipping synchronizing {doc['title']} " - f"since it is environment bound.") + f"since it is environment bound." + ) continue branch_name = f"{args.branch_name}#{doc['type']}" remote_ref = SymbolicReference.create( - repo_to, "refs/remotes/origin/%s" % branch_name) - new_branch = repo_to.create_head(branch_name, 'main') + repo_to, "refs/remotes/origin/%s" % branch_name + ) + new_branch = repo_to.create_head(branch_name, "main") remote_ref = repo_to.remotes[0].refs[branch_name] # get correct type new_branch.set_tracking_branch(remote_ref) new_branch.checkout() @@ -117,7 +115,7 @@ def process_repositories(args, service): pathlib.Path(copy_from, doc["rst_location"]), pathlib.Path(copy_to, doc["rst_location"]), ignore=lambda a, b: ["conf.py"], - dirs_exist_ok=True + dirs_exist_ok=True, ) repo_to.index.add([doc["rst_location"]]) if len(repo_to.index.diff("HEAD")) == 0: @@ -134,53 +132,52 @@ def process_repositories(args, service): ) ) repo_to.remotes.origin.push(new_branch) - if 'github' in url_to and args.open_pr_gh: + if "github" in url_to and args.open_pr_gh: subprocess.run( - args=['gh', 'pr', 'create', '-f'], - cwd=copy_to, - check=True + args=["gh", "pr", "create", "-f"], cwd=copy_to, check=True ) def main(): parser = argparse.ArgumentParser( - description='Synchronize document between environments.') - parser.add_argument( - '--source-environment', - required=True, - help='Environment to be used as a source' + description="Synchronize document between environments." ) parser.add_argument( - '--target-environment', + "--source-environment", required=True, - help='Environment to be used as a source' + help="Environment to be used as a source", ) parser.add_argument( - '--service-type', + "--target-environment", required=True, - help='Service to which document(s) belongs to' + help="Environment to be used as a source", ) parser.add_argument( - '--document-type', + "--service-type", + required=True, + help="Service to which document(s) belongs to", + ) + parser.add_argument( + "--document-type", help=( - 'Type of the document to synchronize. ' - 'All will be synchronized if not set.' - ) + "Type of the document to synchronize. " + "All will be synchronized if not set." + ), ) parser.add_argument( - '--branch-name', + "--branch-name", required=True, - help='Branch name to be used for synchronizing.' + help="Branch name to be used for synchronizing.", ) parser.add_argument( - '--work-dir', + "--work-dir", required=True, - help='Working directory to use for repository checkout.' + help="Working directory to use for repository checkout.", ) parser.add_argument( - '--open-pr-gh', - action='store_true', - help='Open Pull Request using `gh` utility (need to be present).' + "--open-pr-gh", + action="store_true", + help="Open Pull Request using `gh` utility (need to be present).", ) args = parser.parse_args() logging.basicConfig(level=logging.DEBUG) @@ -192,5 +189,5 @@ def main(): process_repositories(args, service) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/update_zuul_project_configs.py b/tools/update_zuul_project_configs.py index 1ed8fa6..4fe5374 100644 --- a/tools/update_zuul_project_configs.py +++ b/tools/update_zuul_project_configs.py @@ -15,15 +15,15 @@ import argparse import logging -import os +# import os import pathlib import requests import subprocess -import warnings +# import warnings from git import exc from git import Repo -from git import SymbolicReference +# from git import SymbolicReference from ruamel.yaml import CommentedMap from ruamel.yaml import YAML @@ -72,7 +72,7 @@ def process_repositories(args, service): workdir.mkdir(exist_ok=True) copy_to = None - repo_to = None + # repo_to = None for repo in service["repositories"]: logging.debug(f"Processing repository {repo}") @@ -208,8 +208,7 @@ def process_repositories(args, service): project["check"]["jobs"].extend( [x["job"]["name"] for x in zuul_new_jobs]) - - #yaml.indent(offset=2, sequence=2) + # yaml.indent(offset=2, sequence=2) with open(zuul_file_name, "w") as f: yaml.dump(zuul_config, f) git_repo.index.add([zuul_file_name.name]) @@ -223,7 +222,8 @@ def process_repositories(args, service): git_repo.index.commit( ( "Update zuul.yaml file\n\n" - "Performed-by: gitea/infra/otc-metadata/tools/update_zuul_project_config.py" + "Performed-by: gitea/infra/otc-metadata" + "/tools/update_zuul_project_config.py" ) ) push_args = ["--set-upstream", "origin", branch_name] diff --git a/tox.ini b/tox.ini index 05782c8..8a1f4d2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,56 +1,26 @@ [tox] minversion = 3.2.0 -envlist = py39,pep8 -skipsdist = True +envlist = py3,pep8 ignore_basepython_conflict = true - [testenv] basepython = python3 usedevelop = True setenv = PYTHONWARNINGS=default::DeprecationWarning - OS_STDOUT_CAPTURE=1 - OS_STDERR_CAPTURE=1 - OS_TEST_TIMEOUT=60 deps = -r{toxinidir}/test-requirements.txt commands = stestr run {posargs} -[testenv:lower-constraints] -deps = -r{toxinidir}/test-requirements.txt - [testenv:pep8] commands = flake8 {posargs} [testenv:venv] commands = {posargs} -[testenv:cover] -setenv = - VIRTUAL_ENV={envdir} - PYTHON=coverage run --source otc_metadata --parallel-mode -commands = - stestr run {posargs} - coverage combine - coverage html -d cover - coverage xml -o cover/coverage.xml - -[testenv:docs] -deps = -r{toxinidir}/doc/requirements.txt -commands = sphinx-build -W -b html doc/source doc/build/html - -[testenv:releasenotes] -deps = {[testenv:docs]deps} -commands = - sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html - -[testenv:debug] -commands = oslo_debug_helper {posargs} - [flake8] # E123, E125 skipped as they are invalid PEP-8. show-source = True -ignore = E123,E125 +ignore = E123,E125,W503 builtins = _ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build diff --git a/zuul.yaml b/zuul.yaml index 901268e..809de6c 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -1,19 +1,11 @@ - project: merge-mode: squash-merge - default-branch: master - templates: - - publish-to-pypi - - publish-otc-docs-hc-pti - - release-notes-jobs + default-branch: main check: jobs: - otc-tox-pep8 - - otc-tox-py38 - otc-tox-py39 - - tox-functional gate: jobs: - otc-tox-pep8 - - otc-tox-py38 - - otc-tox-py39 - - tox-functional + - otc-tox-py39 \ No newline at end of file