From 92a057cb1b747845c5cb7e0ae992acdde87970b8 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Tue, 20 Sep 2022 07:41:00 +0000 Subject: [PATCH 01/16] Repo+Pull --- tools/attention_list.py | 95 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tools/attention_list.py diff --git a/tools/attention_list.py b/tools/attention_list.py new file mode 100644 index 00000000..daeb9c15 --- /dev/null +++ b/tools/attention_list.py @@ -0,0 +1,95 @@ +#!/usr/bin/python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import requests +import json + + +def main(): + parser = argparse.ArgumentParser( + description='Bootstrap repositories.') + parser.add_argument( + '--orgs', + nargs='+', + default=['docs'], + help='One or more organizations to be queried for failed Pull ' + 'Requests.\n' + 'Default: [docs]' + ) + parser.add_argument( + '--token', + default='682ac3d176cf614cec4ecf9c8392880c89d07488', + help='API Token' + ) + parser.add_argument( + '--url', + default='https://gitea.eco.tsi-dev.otc-service.com/api/v1/', + help='Base URL for API request.' + ) + + args = parser.parse_args() + # logging.basicConfig(level=logging.DEBUG) + path = 'orgs/docs/repos?limit=50&page=' + headers = {} + headers['accept'] = 'application/json' + headers['Authorization'] = 'token ' + args.token + repositories = [] + i = 1 + + while True: + try: + url = args.url + path + str(i) + res = requests.request('GET', url=url, headers=headers) + if res.json(): + for repo in res.json(): + repositories.append(repo) + i+=1 + continue + else: + break + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) + break + + pull_path = 'repos/docs/' + pulls = [] + + for repo in repositories: + try: + url = args.url + pull_path + repo['name'] + '/pulls?state=open' + res = requests.request('GET', url=url, headers=headers) + if res.json(): + for pull in res.json(): + pulls.append(pull) + continue + else: + continue + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) + break + + # print(len(pulls)) + + + + + + +if __name__ == '__main__': + main() \ No newline at end of file -- 2.34.1 From 9bb53300680a0e14a3a9ee4cf990c372f95674f5 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Tue, 20 Sep 2022 08:24:06 +0000 Subject: [PATCH 02/16] Failure list --- tools/attention_list.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index daeb9c15..8e03c9e8 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -67,7 +67,9 @@ def main(): break pull_path = 'repos/docs/' + status_path = 'repos/docs/' pulls = [] + failed_commits = [] for repo in repositories: try: @@ -76,6 +78,28 @@ def main(): if res.json(): for pull in res.json(): pulls.append(pull) + + for pull in pulls: + try: + url = args.url + status_path + repo['name'] + '/commits/' + pull['head']['ref'] + '/statuses?limit=1' + res_sta = requests.request('GET', url=url, headers=headers) + if res_sta.json(): + if res_sta.json()[0]['status'] == 'failure': + # print("Pull Request " + pull['url'] + " has a failed check!") + failed_commits.append({ + 'url': pull['url'], + 'status': res_sta.json()[0]['status'], + 'target_url': res_sta.json()[0]['target_url'], + 'created_at': pull['created_at'], + 'updated_at': res_sta.json()[0]['updated_at'] + }) + continue + else: + continue + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) + break continue else: continue @@ -84,7 +108,12 @@ def main(): print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) break - # print(len(pulls)) + + print(json.dumps(failed_commits)) + + + + -- 2.34.1 From b790807bd3a3816a5ae23e76d6fbbbcfec2fd086 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Tue, 20 Sep 2022 08:25:40 +0000 Subject: [PATCH 03/16] Removed token + made it mandatory --- tools/attention_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 8e03c9e8..16e4c667 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -32,7 +32,7 @@ def main(): ) parser.add_argument( '--token', - default='682ac3d176cf614cec4ecf9c8392880c89d07488', + required=True, help='API Token' ) parser.add_argument( -- 2.34.1 From 2b9f70d55916d2bd41653381f5a0d3ad951092f3 Mon Sep 17 00:00:00 2001 From: tischrei Date: Tue, 20 Sep 2022 09:05:48 +0000 Subject: [PATCH 04/16] add methods --- tools/attention_list.py | 51 ++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 16e4c667..a5344922 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -19,7 +19,7 @@ import requests import json -def main(): +def get_args(): parser = argparse.ArgumentParser( description='Bootstrap repositories.') parser.add_argument( @@ -35,25 +35,26 @@ def main(): required=True, help='API Token' ) + parser.add_argument( + '--debug', + action='store_true', + help='Set debug mode on.' + ) parser.add_argument( '--url', default='https://gitea.eco.tsi-dev.otc-service.com/api/v1/', help='Base URL for API request.' ) + return(parser.parse_args()) - args = parser.parse_args() - # logging.basicConfig(level=logging.DEBUG) - path = 'orgs/docs/repos?limit=50&page=' - headers = {} - headers['accept'] = 'application/json' - headers['Authorization'] = 'token ' + args.token +def get_repos(url, path, headers): repositories = [] i = 1 while True: try: - url = args.url + path + str(i) - res = requests.request('GET', url=url, headers=headers) + req_url = url + path + str(i) + res = requests.request('GET', url=req_url, headers=headers) if res.json(): for repo in res.json(): repositories.append(repo) @@ -65,11 +66,13 @@ def main(): print("An error has occured: " + str(e)) print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) break + return(repositories) - pull_path = 'repos/docs/' - status_path = 'repos/docs/' - pulls = [] +def get_failed_commits(repositories, headers, args): failed_commits = [] + pull_path = 'repos/docs/' + status_path = pull_path + pulls = [] for repo in repositories: try: @@ -107,18 +110,24 @@ def main(): print("An error has occured: " + str(e)) print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) break + return(failed_commits) +def main(): + + args = get_args() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + + repo_path = 'orgs/docs/repos?limit=50&page=' + headers = {} + headers['accept'] = 'application/json' + headers['Authorization'] = 'token ' + args.token + + repositories = get_repos(url=args.url, path=repo_path, headers=headers) + failed_commits = get_failed_commits(repositories=repositories, headers=headers, args=args) print(json.dumps(failed_commits)) - - - - - - - - - if __name__ == '__main__': main() \ No newline at end of file -- 2.34.1 From b71f45c846d7fa89cd4cf24419396e75a83870f4 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Wed, 21 Sep 2022 09:13:54 +0000 Subject: [PATCH 05/16] GitHub first try --- tools/attention_list.py | 86 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index a5344922..353aaa38 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -45,6 +45,13 @@ def get_args(): default='https://gitea.eco.tsi-dev.otc-service.com/api/v1/', help='Base URL for API request.' ) + parser.add_argument( + '--hoster', + default='gitea', + choices=['gitea', 'github'], + nargs='?', + help='Hoster for the API calls. Choose between \"github\" and \"gitea\".' + ) return(parser.parse_args()) def get_repos(url, path, headers): @@ -112,6 +119,48 @@ def get_failed_commits(repositories, headers, args): break return(failed_commits) +def get_github_repos(url, headers, org): + repositories = [] + i = 1 + + while True: + try: + req_url = url + 'orgs/' + org + '/repos?page=' + str(i) + res = requests.request('GET', url=req_url, headers=headers) + if res.json(): + for repo in res.json(): + repositories.append(repo) + i+=1 + continue + else: + break + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) + break + return(repositories) + +def get_github_prs(url, headers, org, repo): + pullrequests = [] + i = 1 + + while True: + try: + req_url = url + 'repos/' + org + '/' + repo + '/pulls?state=open&page=' + str(i) + res = requests.request('GET', url=req_url, headers=headers) + if res.json(): + for pr in res.json(): + pullrequests.append(pr) + i+=1 + continue + else: + break + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) + break + return(pullrequests) + def main(): @@ -120,14 +169,37 @@ def main(): if args.debug: logging.basicConfig(level=logging.DEBUG) - repo_path = 'orgs/docs/repos?limit=50&page=' - headers = {} - headers['accept'] = 'application/json' - headers['Authorization'] = 'token ' + args.token + if args.hoster == 'gitea': + repo_path = 'orgs/docs/repos?limit=50&page=' + headers = {} + headers['accept'] = 'application/json' + headers['Authorization'] = 'token ' + args.token + + repositories = get_repos(url=args.url, path=repo_path, headers=headers) + failed_commits = get_failed_commits(repositories=repositories, headers=headers, args=args) + print(json.dumps(failed_commits)) + + elif args.hoster == 'github': + url = 'https://api.github.com/' + headers = {} + headers['accept'] = 'application/json' + headers['Authorization'] = 'Bearer ' + args.token + + + for org in args.orgs: + repos = get_github_repos(url, headers, org) + pulls = [] + for repo in repos: + pulls.extend(get_github_prs(url, headers, org, repo['name'])) + # print(pulls) + print(len(pulls)) + + + + + + - repositories = get_repos(url=args.url, path=repo_path, headers=headers) - failed_commits = get_failed_commits(repositories=repositories, headers=headers, args=args) - print(json.dumps(failed_commits)) if __name__ == '__main__': main() \ No newline at end of file -- 2.34.1 From 5466630a414480668ff12017db7bc99f583eac6a Mon Sep 17 00:00:00 2001 From: tischrei Date: Wed, 21 Sep 2022 12:46:33 +0000 Subject: [PATCH 06/16] first time successful run --- tools/attention_list.py | 56 +++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 353aaa38..3d559f9a 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -161,11 +161,47 @@ def get_github_prs(url, headers, org, repo): break return(pullrequests) +def get_failed_gh_commits(pulls, url, org, repo, headers): + failed_commits = [] + for pull in pulls: + try: + req_url = url + 'repos/' + org + '/' + repo['name'] + '/commits/' + pull['head']['sha'] + '/check-runs' + res_sta = requests.request('GET', url=req_url, headers=headers) + if res_sta.json(): + if len(res_sta.json()['check_runs']) != 0: + if res_sta.json()['check_runs'][0]['conclusion'] == 'failure': + failed_commits.append({ + 'url': pull['html_url'], + 'status': res_sta.json()['check_runs'][0]['conclusion'], + 'target_url': res_sta.json()['check_runs'][0]['details_url'], + 'created_at': pull['created_at'], + 'updated_at': res_sta.json()['check_runs'][0]['completed_at'] + }) + continue + else: + failed_commits.append({ + 'url': pull['html_url'], + 'status': 'undefined', + 'target_url': 'undefined', + 'created_at': pull['created_at'], + 'updated_at': pull['updated_at'] + }) + + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) + print(json.dumps(res_sta.json())) + exit() + break + return(failed_commits) + def main(): args = get_args() + failed_commits = [] + if args.debug: logging.basicConfig(level=logging.DEBUG) @@ -185,21 +221,21 @@ def main(): headers['accept'] = 'application/json' headers['Authorization'] = 'Bearer ' + args.token - for org in args.orgs: repos = get_github_repos(url, headers, org) - pulls = [] for repo in repos: - pulls.extend(get_github_prs(url, headers, org, repo['name'])) - # print(pulls) - print(len(pulls)) - - + pulls = get_github_prs(url, headers, org, repo['name']) + if pulls: + commits = get_failed_gh_commits( + pulls=pulls, + url=url, + org=org, + repo=repo, + headers=headers) + failed_commits.extend(commits) - - - + print(json.dumps(failed_commits)) if __name__ == '__main__': main() \ No newline at end of file -- 2.34.1 From 78184c521796071bd24b523294fd752518917c8e Mon Sep 17 00:00:00 2001 From: tischrei Date: Mon, 26 Sep 2022 14:07:55 +0000 Subject: [PATCH 07/16] some changes --- tools/attention_list.py | 129 +++++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 3d559f9a..3475b50a 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -17,6 +17,7 @@ import argparse import logging import requests import json +import threading def get_args(): @@ -31,9 +32,12 @@ def get_args(): 'Default: [docs]' ) parser.add_argument( - '--token', - required=True, - help='API Token' + '--gh-token', + help='API Token for GitHub.' + ) + parser.add_argument( + '--gitea-token', + help='API Token for Gitea' ) parser.add_argument( '--debug', @@ -47,10 +51,10 @@ def get_args(): ) parser.add_argument( '--hoster', - default='gitea', - choices=['gitea', 'github'], - nargs='?', - help='Hoster for the API calls. Choose between \"github\" and \"gitea\".' + default=['gitea', 'github'], + nargs='+', + help='Git hoster to be queried for failed Pull Requests.\n + 'Default: [github, gitea].' ) return(parser.parse_args()) @@ -75,7 +79,7 @@ def get_repos(url, path, headers): break return(repositories) -def get_failed_commits(repositories, headers, args): +def get_failed_gitea_commits(repositories, headers, args): failed_commits = [] pull_path = 'repos/docs/' status_path = pull_path @@ -161,38 +165,37 @@ def get_github_prs(url, headers, org, repo): break return(pullrequests) -def get_failed_gh_commits(pulls, url, org, repo, headers): +def get_failed_gh_commits(pull, url, org, repo, headers): failed_commits = [] - for pull in pulls: - try: - req_url = url + 'repos/' + org + '/' + repo['name'] + '/commits/' + pull['head']['sha'] + '/check-runs' - res_sta = requests.request('GET', url=req_url, headers=headers) - if res_sta.json(): - if len(res_sta.json()['check_runs']) != 0: - if res_sta.json()['check_runs'][0]['conclusion'] == 'failure': - failed_commits.append({ - 'url': pull['html_url'], - 'status': res_sta.json()['check_runs'][0]['conclusion'], - 'target_url': res_sta.json()['check_runs'][0]['details_url'], - 'created_at': pull['created_at'], - 'updated_at': res_sta.json()['check_runs'][0]['completed_at'] - }) - continue - else: + try: + req_url = url + 'repos/' + org + '/' + repo['name'] + '/commits/' + pull['head']['sha'] + '/check-runs' + res_sta = requests.request('GET', url=req_url, headers=headers) + if res_sta.json(): + if len(res_sta.json()['check_runs']) != 0: + if res_sta.json()['check_runs'][0]['conclusion'] == 'failure': failed_commits.append({ 'url': pull['html_url'], - 'status': 'undefined', - 'target_url': 'undefined', + 'status': res_sta.json()['check_runs'][0]['conclusion'], + 'target_url': res_sta.json()['check_runs'][0]['details_url'], 'created_at': pull['created_at'], - 'updated_at': pull['updated_at'] + 'updated_at': res_sta.json()['check_runs'][0]['completed_at'] }) + continue + else: + failed_commits.append({ + 'url': pull['html_url'], + 'status': 'undefined', + 'target_url': 'undefined', + 'created_at': pull['created_at'], + 'updated_at': pull['updated_at'] + }) - except Exception as e: - print("An error has occured: " + str(e)) - print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) - print(json.dumps(res_sta.json())) - exit() - break + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) + print(json.dumps(res_sta.json())) + exit() + break return(failed_commits) @@ -205,34 +208,40 @@ def main(): if args.debug: logging.basicConfig(level=logging.DEBUG) - if args.hoster == 'gitea': - repo_path = 'orgs/docs/repos?limit=50&page=' - headers = {} - headers['accept'] = 'application/json' - headers['Authorization'] = 'token ' + args.token + for h in args.hoster: + if h == 'gitea': + repo_path = 'orgs/docs/repos?limit=50&page=' + headers = {} + headers['accept'] = 'application/json' + if not args.gitea_token: + raise Exception('Please, provide Gitea Token as argument.') + headers['Authorization'] = 'token ' + args.gitea_token - repositories = get_repos(url=args.url, path=repo_path, headers=headers) - failed_commits = get_failed_commits(repositories=repositories, headers=headers, args=args) - print(json.dumps(failed_commits)) - - elif args.hoster == 'github': - url = 'https://api.github.com/' - headers = {} - headers['accept'] = 'application/json' - headers['Authorization'] = 'Bearer ' + args.token + repositories = get_repos(url=args.url, path=repo_path, headers=headers) + commits = get_failed_gitea_commits(repositories=repositories, headers=headers, args=args) + failed_commits.extend(commits) + + elif h == 'github': + url = 'https://api.github.com/' + headers = {} + headers['accept'] = 'application/json' + if not args.gh_token: + raise Exception('Please, provide GitHub Token as argument.') + headers['Authorization'] = 'Bearer ' + args.gh_token - for org in args.orgs: - repos = get_github_repos(url, headers, org) - for repo in repos: - pulls = get_github_prs(url, headers, org, repo['name']) - if pulls: - commits = get_failed_gh_commits( - pulls=pulls, - url=url, - org=org, - repo=repo, - headers=headers) - failed_commits.extend(commits) + for org in args.orgs: + repos = get_github_repos(url, headers, org) + for repo in repos: + pulls = get_github_prs(url, headers, org, repo['name']) + if pulls: + for pull in pulls: + commits = get_failed_gh_commits( + pull=pull, + url=url, + org=org, + repo=repo, + headers=headers) + failed_commits.extend(commits) print(json.dumps(failed_commits)) -- 2.34.1 From be58f093a47ca486d4287ce8421532d2461d48a1 Mon Sep 17 00:00:00 2001 From: tischrei Date: Tue, 27 Sep 2022 08:07:54 +0000 Subject: [PATCH 08/16] update to class --- tools/attention_list.py | 84 +++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 3475b50a..2698c212 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -19,15 +19,33 @@ import requests import json import threading +class FailedPR: + def __init__(self, url, status): + self.url = url + self.status = status + self.target_url = target_url + self.created_at = created_at + self.updated_at = updated_at + + def to_json(self): + return json.dumps(self.__dict__) def get_args(): parser = argparse.ArgumentParser( description='Bootstrap repositories.') parser.add_argument( - '--orgs', + '--gh-orgs', + nargs='+', + default=['opentelekomcloud-docs'], + help='One or more GitHub organizations to be queried for failed Pull ' + 'Requests.\n' + 'Default: [opentelekomcloud-docs]' + ) + parser.add_argument( + '--gitea-orgs', nargs='+', default=['docs'], - help='One or more organizations to be queried for failed Pull ' + help='One or more Gitea organizations to be queried for failed Pull ' 'Requests.\n' 'Default: [docs]' ) @@ -45,15 +63,16 @@ def get_args(): help='Set debug mode on.' ) parser.add_argument( - '--url', + '--gitea-url', default='https://gitea.eco.tsi-dev.otc-service.com/api/v1/', - help='Base URL for API request.' + help='Base URL for API request.\n' + 'Default: https://gitea.eco.tsi-dev.otc-service.com/api/v1/' ) parser.add_argument( '--hoster', default=['gitea', 'github'], nargs='+', - help='Git hoster to be queried for failed Pull Requests.\n + help='Git hoster to be queried for failed Pull Requests.\n' 'Default: [github, gitea].' ) return(parser.parse_args()) @@ -99,7 +118,6 @@ def get_failed_gitea_commits(repositories, headers, args): res_sta = requests.request('GET', url=url, headers=headers) if res_sta.json(): if res_sta.json()[0]['status'] == 'failure': - # print("Pull Request " + pull['url'] + " has a failed check!") failed_commits.append({ 'url': pull['url'], 'status': res_sta.json()[0]['status'], @@ -123,13 +141,13 @@ def get_failed_gitea_commits(repositories, headers, args): break return(failed_commits) -def get_github_repos(url, headers, org): +def get_github_repos(url, headers, gh_org): repositories = [] i = 1 while True: try: - req_url = url + 'orgs/' + org + '/repos?page=' + str(i) + req_url = url + 'orgs/' + gh_org + '/repos?page=' + str(i) res = requests.request('GET', url=req_url, headers=headers) if res.json(): for repo in res.json(): @@ -144,13 +162,13 @@ def get_github_repos(url, headers, org): break return(repositories) -def get_github_prs(url, headers, org, repo): +def get_github_prs(url, headers, gh_org, repo): pullrequests = [] i = 1 while True: try: - req_url = url + 'repos/' + org + '/' + repo + '/pulls?state=open&page=' + str(i) + req_url = url + 'repos/' + gh_org + '/' + repo + '/pulls?state=open&page=' + str(i) res = requests.request('GET', url=req_url, headers=headers) if res.json(): for pr in res.json(): @@ -165,37 +183,37 @@ def get_github_prs(url, headers, org, repo): break return(pullrequests) -def get_failed_gh_commits(pull, url, org, repo, headers): +def get_failed_gh_commits(pull, url, gh_org, repo, headers): failed_commits = [] try: - req_url = url + 'repos/' + org + '/' + repo['name'] + '/commits/' + pull['head']['sha'] + '/check-runs' + req_url = url + 'repos/' + gh_org + '/' + repo['name'] + '/commits/' + pull['head']['sha'] + '/check-runs' res_sta = requests.request('GET', url=req_url, headers=headers) if res_sta.json(): if len(res_sta.json()['check_runs']) != 0: if res_sta.json()['check_runs'][0]['conclusion'] == 'failure': - failed_commits.append({ - 'url': pull['html_url'], - 'status': res_sta.json()['check_runs'][0]['conclusion'], - 'target_url': res_sta.json()['check_runs'][0]['details_url'], - 'created_at': pull['created_at'], - 'updated_at': res_sta.json()['check_runs'][0]['completed_at'] - }) - continue + o = FailedPR( + url=pull['html_url'], + status=res_sta.json()['check_runs'][0]['conclusion'], + target_url=res_sta.json()['check_runs'][0]['details_url'], + created_at=pull['created_at'], + updated_at=res_sta.json()['check_runs'][0]['completed_at'] + ) + failed_commits.append(o) else: - failed_commits.append({ - 'url': pull['html_url'], - 'status': 'undefined', - 'target_url': 'undefined', - 'created_at': pull['created_at'], - 'updated_at': pull['updated_at'] - }) + o = FailedPR( + url=pull['html_url'], + status='', + target_url='', + created_at=pull['created_at'], + updated_at=res_sta.json()['check_runs'][0]['completed_at'] + ) + failed_commits.append(o) except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) print(json.dumps(res_sta.json())) exit() - break return(failed_commits) @@ -217,7 +235,7 @@ def main(): raise Exception('Please, provide Gitea Token as argument.') headers['Authorization'] = 'token ' + args.gitea_token - repositories = get_repos(url=args.url, path=repo_path, headers=headers) + repositories = get_repos(url=args.gitea_url, path=repo_path, headers=headers) commits = get_failed_gitea_commits(repositories=repositories, headers=headers, args=args) failed_commits.extend(commits) @@ -229,16 +247,16 @@ def main(): raise Exception('Please, provide GitHub Token as argument.') headers['Authorization'] = 'Bearer ' + args.gh_token - for org in args.orgs: - repos = get_github_repos(url, headers, org) + for org in args.gh_orgs: + repos = get_github_repos(url=url, headers=headers, gh_org=org) for repo in repos: - pulls = get_github_prs(url, headers, org, repo['name']) + pulls = get_github_prs(url=url, headers=headers, gh_org=org, repo=repo['name']) if pulls: for pull in pulls: commits = get_failed_gh_commits( pull=pull, url=url, - org=org, + gh_org=org, repo=repo, headers=headers) failed_commits.extend(commits) -- 2.34.1 From 127642128eff3fd111c25212e02726ea84ee0655 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Tue, 27 Sep 2022 09:52:27 +0000 Subject: [PATCH 09/16] Refactored gitea integration --- tools/attention_list.py | 164 +++++++++++++++++++++++++--------------- 1 file changed, 104 insertions(+), 60 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 2698c212..40a06eb3 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -20,7 +20,8 @@ import json import threading class FailedPR: - def __init__(self, url, status): + def __init__(self, host, url, status, target_url, created_at, updated_at): + self.host = host self.url = url self.status = status self.target_url = target_url @@ -77,68 +78,70 @@ def get_args(): ) return(parser.parse_args()) -def get_repos(url, path, headers): +def get_gitea_repos(url, headers, gitea_org): repositories = [] i = 1 while True: try: - req_url = url + path + str(i) + req_url = (url + 'orgs/' + gitea_org + + '/repos?limit=50&page=' + str(i)) res = requests.request('GET', url=req_url, headers=headers) + i+=1 if res.json(): for repo in res.json(): repositories.append(repo) - i+=1 continue else: break except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) + print("The request status is: " + str(res.status_code) + + " | " + str(res.reason)) break return(repositories) -def get_failed_gitea_commits(repositories, headers, args): +def get_gitea_prs(url, headers, gitea_org, repo): + pullrequests = [] + + try: + req_url = (url + 'repos/' + gitea_org + '/' + + repo + '/pulls?state=open') + res = requests.request('GET', url=req_url, headers=headers) + if res.json(): + for pr in res.json(): + pullrequests.append(pr) + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res.status_code) + + " | " + str(res.reason)) + exit() + return(pullrequests) + +def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): failed_commits = [] - pull_path = 'repos/docs/' - status_path = pull_path - pulls = [] + try: + req_url = (url + 'repos/' + gitea_org + '/' + repo['name'] + + '/commits/' + pull['head']['ref'] + '/statuses?limit=1') + res_sta = requests.request('GET', url=req_url, headers=headers) + if res_sta.json(): + if res_sta.json()[0]['status'] == 'failure': + o = FailedPR( + host='gitea', + url=pull['url'], + status=res_sta.json()[0]['status'], + target_url=res_sta.json()[0]['target_url'], + created_at=pull['created_at'], + updated_at=res_sta.json()[0]['updated_at'] + ) + failed_commits.append(o) - for repo in repositories: - try: - url = args.url + pull_path + repo['name'] + '/pulls?state=open' - res = requests.request('GET', url=url, headers=headers) - if res.json(): - for pull in res.json(): - pulls.append(pull) - - for pull in pulls: - try: - url = args.url + status_path + repo['name'] + '/commits/' + pull['head']['ref'] + '/statuses?limit=1' - res_sta = requests.request('GET', url=url, headers=headers) - if res_sta.json(): - if res_sta.json()[0]['status'] == 'failure': - failed_commits.append({ - 'url': pull['url'], - 'status': res_sta.json()[0]['status'], - 'target_url': res_sta.json()[0]['target_url'], - 'created_at': pull['created_at'], - 'updated_at': res_sta.json()[0]['updated_at'] - }) - continue - else: - continue - except Exception as e: - print("An error has occured: " + str(e)) - print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) - break - continue - else: - continue - except Exception as e: - print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) - break + except Exception as e: + print("An error has occured: " + str(e)) + print("The request status is: " + str(res_sta.status_code) + + " | " + str(res_sta.reason)) + print(json.dumps(res_sta.json())) + exit() return(failed_commits) def get_github_repos(url, headers, gh_org): @@ -158,7 +161,8 @@ def get_github_repos(url, headers, gh_org): break except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) + print("The request status is: " + str(res.status_code) + + " | " + str(res.reason)) break return(repositories) @@ -168,7 +172,8 @@ def get_github_prs(url, headers, gh_org, repo): while True: try: - req_url = url + 'repos/' + gh_org + '/' + repo + '/pulls?state=open&page=' + str(i) + req_url = (url + 'repos/' + gh_org + '/' + repo + + '/pulls?state=open&page=' + str(i)) res = requests.request('GET', url=req_url, headers=headers) if res.json(): for pr in res.json(): @@ -179,28 +184,34 @@ def get_github_prs(url, headers, gh_org, repo): break except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) + print("The request status is: " + str(res.status_code) + + " | " + str(res.reason)) break return(pullrequests) def get_failed_gh_commits(pull, url, gh_org, repo, headers): failed_commits = [] try: - req_url = url + 'repos/' + gh_org + '/' + repo['name'] + '/commits/' + pull['head']['sha'] + '/check-runs' + req_url = (url + 'repos/' + gh_org + '/' + repo['name'] + + '/commits/' + pull['head']['sha'] + '/check-runs') res_sta = requests.request('GET', url=req_url, headers=headers) if res_sta.json(): if len(res_sta.json()['check_runs']) != 0: if res_sta.json()['check_runs'][0]['conclusion'] == 'failure': o = FailedPR( + host='gitea', url=pull['html_url'], status=res_sta.json()['check_runs'][0]['conclusion'], - target_url=res_sta.json()['check_runs'][0]['details_url'], + target_url=(res_sta.json()['check_runs'] + [0]['details_url']), created_at=pull['created_at'], - updated_at=res_sta.json()['check_runs'][0]['completed_at'] + updated_at=(res_sta.json()['check_runs'] + [0]['completed_at']) ) failed_commits.append(o) else: o = FailedPR( + host='gitea', url=pull['html_url'], status='', target_url='', @@ -211,7 +222,8 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) + print("The request status is: " + str(res_sta.status_code) + + " | " + str(res_sta.reason)) print(json.dumps(res_sta.json())) exit() return(failed_commits) @@ -228,16 +240,38 @@ def main(): for h in args.hoster: if h == 'gitea': - repo_path = 'orgs/docs/repos?limit=50&page=' + headers = {} headers['accept'] = 'application/json' if not args.gitea_token: raise Exception('Please, provide Gitea Token as argument.') + if not args.gitea_url: + raise Exception('Please, provide Gitea URL as argument.') headers['Authorization'] = 'token ' + args.gitea_token - repositories = get_repos(url=args.gitea_url, path=repo_path, headers=headers) - commits = get_failed_gitea_commits(repositories=repositories, headers=headers, args=args) - failed_commits.extend(commits) + for org in args.gitea_orgs: + repos = get_gitea_repos( + url=args.gitea_url, + headers=headers, + gitea_org=org + ) + for repo in repos: + pulls = get_gitea_prs( + url=args.gitea_url, + headers=headers, + gitea_org=org, + repo=repo['name'] + ) + if pulls: + for pull in pulls: + commits = get_failed_gitea_commits( + pull=pull, + url=args.gitea_url, + gitea_org=org, + repo=repo, + headers=headers + ) + failed_commits.extend(commits) elif h == 'github': url = 'https://api.github.com/' @@ -248,9 +282,18 @@ def main(): headers['Authorization'] = 'Bearer ' + args.gh_token for org in args.gh_orgs: - repos = get_github_repos(url=url, headers=headers, gh_org=org) + repos = get_github_repos( + url=url, + headers=headers, + gh_org=org + ) for repo in repos: - pulls = get_github_prs(url=url, headers=headers, gh_org=org, repo=repo['name']) + pulls = get_github_prs( + url=url, + headers=headers, + gh_org=org, + repo=repo['name'] + ) if pulls: for pull in pulls: commits = get_failed_gh_commits( @@ -258,11 +301,12 @@ def main(): url=url, gh_org=org, repo=repo, - headers=headers) + headers=headers + ) failed_commits.extend(commits) - - print(json.dumps(failed_commits)) + for o in failed_commits: + print(o.to_json()) if __name__ == '__main__': main() \ No newline at end of file -- 2.34.1 From 8c13133c602f2afc9dc43113453373dfa4aac62a Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Tue, 27 Sep 2022 10:20:14 +0000 Subject: [PATCH 10/16] Some pep8 fixes --- tools/attention_list.py | 45 +++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 40a06eb3..3cc0c32e 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -27,7 +27,7 @@ class FailedPR: self.target_url = target_url self.created_at = created_at self.updated_at = updated_at - + def to_json(self): return json.dumps(self.__dict__) @@ -76,7 +76,7 @@ def get_args(): help='Git hoster to be queried for failed Pull Requests.\n' 'Default: [github, gitea].' ) - return(parser.parse_args()) + return parser.parse_args() def get_gitea_repos(url, headers, gitea_org): repositories = [] @@ -84,7 +84,7 @@ def get_gitea_repos(url, headers, gitea_org): while True: try: - req_url = (url + 'orgs/' + gitea_org + + req_url = (url + 'orgs/' + gitea_org + '/repos?limit=50&page=' + str(i)) res = requests.request('GET', url=req_url, headers=headers) i+=1 @@ -96,16 +96,16 @@ def get_gitea_repos(url, headers, gitea_org): break except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) break - return(repositories) + return repositories def get_gitea_prs(url, headers, gitea_org, repo): pullrequests = [] try: - req_url = (url + 'repos/' + gitea_org + '/' + req_url = (url + 'repos/' + gitea_org + '/' + repo + '/pulls?state=open') res = requests.request('GET', url=req_url, headers=headers) if res.json(): @@ -113,15 +113,15 @@ def get_gitea_prs(url, headers, gitea_org, repo): pullrequests.append(pr) except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) exit() - return(pullrequests) + return pullrequests def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): failed_commits = [] try: - req_url = (url + 'repos/' + gitea_org + '/' + repo['name'] + + req_url = (url + 'repos/' + gitea_org + '/' + repo['name'] + '/commits/' + pull['head']['ref'] + '/statuses?limit=1') res_sta = requests.request('GET', url=req_url, headers=headers) if res_sta.json(): @@ -138,11 +138,11 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res_sta.status_code) + + print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) print(json.dumps(res_sta.json())) exit() - return(failed_commits) + return failed_commits def get_github_repos(url, headers, gh_org): repositories = [] @@ -161,10 +161,10 @@ def get_github_repos(url, headers, gh_org): break except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) break - return(repositories) + return repositories def get_github_prs(url, headers, gh_org, repo): pullrequests = [] @@ -184,15 +184,15 @@ def get_github_prs(url, headers, gh_org, repo): break except Exception as e: print("An error has occured: " + str(e)) - print("The request status is: " + str(res.status_code) + + print("The request status is: " + str(res.status_code) + " | " + str(res.reason)) break - return(pullrequests) + return pullrequests def get_failed_gh_commits(pull, url, gh_org, repo, headers): failed_commits = [] try: - req_url = (url + 'repos/' + gh_org + '/' + repo['name'] + + req_url = (url + 'repos/' + gh_org + '/' + repo['name'] + '/commits/' + pull['head']['sha'] + '/check-runs') res_sta = requests.request('GET', url=req_url, headers=headers) if res_sta.json(): @@ -226,21 +226,18 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): " | " + str(res_sta.reason)) print(json.dumps(res_sta.json())) exit() - return(failed_commits) - + return failed_commits def main(): - args = get_args() - failed_commits = [] if args.debug: logging.basicConfig(level=logging.DEBUG) - + for h in args.hoster: if h == 'gitea': - + headers = {} headers['accept'] = 'application/json' if not args.gitea_token: @@ -272,7 +269,7 @@ def main(): headers=headers ) failed_commits.extend(commits) - + elif h == 'github': url = 'https://api.github.com/' headers = {} @@ -309,4 +306,4 @@ def main(): print(o.to_json()) if __name__ == '__main__': - main() \ No newline at end of file + main() -- 2.34.1 From 841bda3ac792135f5b2dbf894c331b1c6c14f556 Mon Sep 17 00:00:00 2001 From: Sebastian Gode Date: Tue, 27 Sep 2022 10:58:48 +0000 Subject: [PATCH 11/16] spelling mistake --- tools/attention_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 3cc0c32e..d82475b8 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -199,7 +199,7 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): if len(res_sta.json()['check_runs']) != 0: if res_sta.json()['check_runs'][0]['conclusion'] == 'failure': o = FailedPR( - host='gitea', + host='github', url=pull['html_url'], status=res_sta.json()['check_runs'][0]['conclusion'], target_url=(res_sta.json()['check_runs'] @@ -211,7 +211,7 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): failed_commits.append(o) else: o = FailedPR( - host='gitea', + host='github', url=pull['html_url'], status='', target_url='', -- 2.34.1 From 9539863e5f0538346380a3861e02e75612055b5f Mon Sep 17 00:00:00 2001 From: tischrei Date: Tue, 27 Sep 2022 11:49:37 +0000 Subject: [PATCH 12/16] json return and error codes --- tools/attention_list.py | 49 +++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index d82475b8..470019ff 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Error Codes +# 1000: Failed check (default error) +# 1001: Check not existing but expected + import argparse import logging import requests @@ -20,16 +24,16 @@ import json import threading class FailedPR: - def __init__(self, host, url, status, target_url, created_at, updated_at): + + def __init__(self, host, url, created_at, updated_at, target_url=None, status=None, error=None): self.host = host self.url = url self.status = status self.target_url = target_url self.created_at = created_at self.updated_at = updated_at + self.error = error - def to_json(self): - return json.dumps(self.__dict__) def get_args(): parser = argparse.ArgumentParser( @@ -66,9 +70,15 @@ def get_args(): parser.add_argument( '--gitea-url', default='https://gitea.eco.tsi-dev.otc-service.com/api/v1/', - help='Base URL for API request.\n' + help='Gitea base URL for API request.\n' 'Default: https://gitea.eco.tsi-dev.otc-service.com/api/v1/' ) + parser.add_argument( + '--gh-url', + default='https://api.github.com/', + help='GitHub Base URL for API request.\n' + 'Default: https://api.github.com/' + ) parser.add_argument( '--hoster', default=['gitea', 'github'], @@ -132,7 +142,8 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): status=res_sta.json()[0]['status'], target_url=res_sta.json()[0]['target_url'], created_at=pull['created_at'], - updated_at=res_sta.json()[0]['updated_at'] + updated_at=res_sta.json()[0]['updated_at'], + error=1000 ) failed_commits.append(o) @@ -206,17 +217,17 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): [0]['details_url']), created_at=pull['created_at'], updated_at=(res_sta.json()['check_runs'] - [0]['completed_at']) + [0]['completed_at']), + error=1000 ) failed_commits.append(o) else: o = FailedPR( host='github', url=pull['html_url'], - status='', - target_url='', created_at=pull['created_at'], - updated_at=res_sta.json()['check_runs'][0]['completed_at'] + updated_at=pull['updated_at'], + error=1001, ) failed_commits.append(o) @@ -228,6 +239,17 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): exit() return failed_commits +def create_json_result(failed_commits): + failed_commits_json = [] + for o in failed_commits: + failed_commits_json.append(vars(o)) + result = {} + result['meta'] = {} + result['meta']['count'] = len(failed_commits_json) + result['data'] = failed_commits_json + return json.dumps(result) + + def main(): args = get_args() failed_commits = [] @@ -271,7 +293,9 @@ def main(): failed_commits.extend(commits) elif h == 'github': - url = 'https://api.github.com/' + if not args.gh_url: + raise ValueError('Parameter --gh-url not found.') + url = args.gh_url headers = {} headers['accept'] = 'application/json' if not args.gh_token: @@ -301,9 +325,10 @@ def main(): headers=headers ) failed_commits.extend(commits) + + print(create_json_result( + failed_commits=failed_commits)) - for o in failed_commits: - print(o.to_json()) if __name__ == '__main__': main() -- 2.34.1 From f2fd5494c1d21a20b1c57ec536038a065bd95938 Mon Sep 17 00:00:00 2001 From: tischrei Date: Tue, 27 Sep 2022 12:14:29 +0000 Subject: [PATCH 13/16] add comments --- tools/attention_list.py | 48 +++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 470019ff..eea62343 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -24,18 +24,22 @@ import json import threading class FailedPR: + """Base class for failed Pull Requests""" - def __init__(self, host, url, created_at, updated_at, target_url=None, status=None, error=None): + def __init__(self, host, url, created_at, updated_at, zuul_url=None, status=None, error=None): self.host = host self.url = url self.status = status - self.target_url = target_url + self.zuul_url = zuul_url self.created_at = created_at self.updated_at = updated_at self.error = error def get_args(): + """ + Function to collect parameter. + """ parser = argparse.ArgumentParser( description='Bootstrap repositories.') parser.add_argument( @@ -89,6 +93,9 @@ def get_args(): return parser.parse_args() def get_gitea_repos(url, headers, gitea_org): + """ + Get all Repositories of one Gitea orgainzation + """ repositories = [] i = 1 @@ -112,6 +119,9 @@ def get_gitea_repos(url, headers, gitea_org): return repositories def get_gitea_prs(url, headers, gitea_org, repo): + """ + Collect all Pull Requests of a Gitea Repository + """ pullrequests = [] try: @@ -129,6 +139,9 @@ def get_gitea_prs(url, headers, gitea_org, repo): return pullrequests def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): + """ + Collect all failed gitea commits of one repository. + """ failed_commits = [] try: req_url = (url + 'repos/' + gitea_org + '/' + repo['name'] + @@ -140,7 +153,7 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): host='gitea', url=pull['url'], status=res_sta.json()[0]['status'], - target_url=res_sta.json()[0]['target_url'], + zuul_url=res_sta.json()[0]['target_url'], created_at=pull['created_at'], updated_at=res_sta.json()[0]['updated_at'], error=1000 @@ -156,6 +169,9 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): return failed_commits def get_github_repos(url, headers, gh_org): + """ + Get all repositories of one GitHub organization + """ repositories = [] i = 1 @@ -178,6 +194,9 @@ def get_github_repos(url, headers, gh_org): return repositories def get_github_prs(url, headers, gh_org, repo): + """ + Get all Pull Requests of one GitHub repository + """ pullrequests = [] i = 1 @@ -201,6 +220,9 @@ def get_github_prs(url, headers, gh_org, repo): return pullrequests def get_failed_gh_commits(pull, url, gh_org, repo, headers): + """ + Collect all Failed Pull Requests of one GitHub repository + """ failed_commits = [] try: req_url = (url + 'repos/' + gh_org + '/' + repo['name'] + @@ -213,7 +235,7 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): host='github', url=pull['html_url'], status=res_sta.json()['check_runs'][0]['conclusion'], - target_url=(res_sta.json()['check_runs'] + zuul_url=(res_sta.json()['check_runs'] [0]['details_url']), created_at=pull['created_at'], updated_at=(res_sta.json()['check_runs'] @@ -240,13 +262,21 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): return failed_commits def create_json_result(failed_commits): - failed_commits_json = [] - for o in failed_commits: - failed_commits_json.append(vars(o)) + """ + Create Result + """ result = {} result['meta'] = {} - result['meta']['count'] = len(failed_commits_json) - result['data'] = failed_commits_json + result['data'] = [] + if len(failed_commits) != 0: + failed_commits_json = [] + for o in failed_commits: + failed_commits_json.append(vars(o)) + result['meta']['count'] = len(failed_commits_json) + result['data'] = failed_commits_json + else: + result['meta']['count'] = 0 + return json.dumps(result) -- 2.34.1 From d774cc2e8f504914d9a5f8815133af2cfadc3f3c Mon Sep 17 00:00:00 2001 From: tischrei Date: Tue, 27 Sep 2022 13:24:06 +0000 Subject: [PATCH 14/16] first alpha --- tools/attention_list.py | 60 ++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index eea62343..e03d91b4 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -21,19 +21,34 @@ import argparse import logging import requests import json -import threading + class FailedPR: """Base class for failed Pull Requests""" - def __init__(self, host, url, created_at, updated_at, zuul_url=None, status=None, error=None): - self.host = host - self.url = url - self.status = status - self.zuul_url = zuul_url + def __init__( + self, + created_at, + host, + updated_at, + url, + error=None, + pullrequest=None, + org=None, + repo=None, + status=None, + zuul_url=None): + self.created_at = created_at - self.updated_at = updated_at self.error = error + self.host = host + self.org = org + self.pullrequest = pullrequest + self.repo = repo + self.status = status + self.updated_at = updated_at + self.url = url + self.zuul_url = zuul_url def get_args(): @@ -92,6 +107,7 @@ def get_args(): ) return parser.parse_args() + def get_gitea_repos(url, headers, gitea_org): """ Get all Repositories of one Gitea orgainzation @@ -101,10 +117,13 @@ def get_gitea_repos(url, headers, gitea_org): while True: try: - req_url = (url + 'orgs/' + gitea_org + - '/repos?limit=50&page=' + str(i)) + req_url = (url + + 'orgs/' + + gitea_org + + '/repos?limit=50&page=' + + str(i)) res = requests.request('GET', url=req_url, headers=headers) - i+=1 + i += 1 if res.json(): for repo in res.json(): repositories.append(repo) @@ -114,7 +133,7 @@ def get_gitea_repos(url, headers, gitea_org): except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res.status_code) + - " | " + str(res.reason)) + " | " + str(res.reason)) break return repositories @@ -152,6 +171,9 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): o = FailedPR( host='gitea', url=pull['url'], + org=gitea_org, + repo=repo['name'], + pullrequest=pull['title'], status=res_sta.json()[0]['status'], zuul_url=res_sta.json()[0]['target_url'], created_at=pull['created_at'], @@ -219,6 +241,7 @@ def get_github_prs(url, headers, gh_org, repo): break return pullrequests + def get_failed_gh_commits(pull, url, gh_org, repo, headers): """ Collect all Failed Pull Requests of one GitHub repository @@ -234,6 +257,9 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): o = FailedPR( host='github', url=pull['html_url'], + org=gh_org, + repo=repo['name'], + pullrequest=pull['title'], status=res_sta.json()['check_runs'][0]['conclusion'], zuul_url=(res_sta.json()['check_runs'] [0]['details_url']), @@ -247,6 +273,9 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): o = FailedPR( host='github', url=pull['html_url'], + org=gh_org, + repo=repo['name'], + pullrequest=pull['title'], created_at=pull['created_at'], updated_at=pull['updated_at'], error=1001, @@ -261,6 +290,7 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): exit() return failed_commits + def create_json_result(failed_commits): """ Create Result @@ -278,7 +308,7 @@ def create_json_result(failed_commits): result['meta']['count'] = 0 return json.dumps(result) - + def main(): args = get_args() @@ -355,9 +385,9 @@ def main(): headers=headers ) failed_commits.extend(commits) - - print(create_json_result( - failed_commits=failed_commits)) + + print(create_json_result( + failed_commits=failed_commits)) if __name__ == '__main__': -- 2.34.1 From c660386935d583a39ef3d027cb514bb4a239a9fe Mon Sep 17 00:00:00 2001 From: tischrei Date: Wed, 28 Sep 2022 12:04:32 +0000 Subject: [PATCH 15/16] jobs added and archived pull requests removed --- tools/attention_list.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index e03d91b4..8152e80a 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -21,6 +21,7 @@ import argparse import logging import requests import json +import re class FailedPR: @@ -107,6 +108,29 @@ def get_args(): ) return parser.parse_args() +def add_builds_to_obj(obj, url, tenant): + """ + This method trys to find all build jobs under a Zuul buildset. + The corresponding data like log_url and status will be added. + """ + zuul_api_url = "https://zuul.otc-service.com/api/tenant/" + tenant + "/buildset/" + final_url = re.sub('.*\/buildset\/', zuul_api_url, url) + zuul_headers = {} + zuul_headers['accept'] = 'application/json' + res_zuul = requests.request('GET', url=final_url, headers=zuul_headers) + if res_zuul.status_code != 404 and res_zuul.json(): + x = res_zuul.json() + if len(x['builds']) != 0: + jobs = [] + for build in x['builds']: + job = {} + job['uuid'] = build['uuid'] + job['name'] = build['job_name'] + job['result'] = build['result'] + job['log_url'] = build['log_url'] + jobs.append(job) + obj.jobs = jobs + return obj def get_gitea_repos(url, headers, gitea_org): """ @@ -168,6 +192,7 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): res_sta = requests.request('GET', url=req_url, headers=headers) if res_sta.json(): if res_sta.json()[0]['status'] == 'failure': + zuul_url = res_sta.json()[0]['target_url'] o = FailedPR( host='gitea', url=pull['url'], @@ -175,19 +200,18 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): repo=repo['name'], pullrequest=pull['title'], status=res_sta.json()[0]['status'], - zuul_url=res_sta.json()[0]['target_url'], + zuul_url=zuul_url, created_at=pull['created_at'], updated_at=res_sta.json()[0]['updated_at'], error=1000 ) + o = add_builds_to_obj(obj=o, url=zuul_url, tenant='gl') failed_commits.append(o) except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) - print(json.dumps(res_sta.json())) - exit() return failed_commits def get_github_repos(url, headers, gh_org): @@ -203,7 +227,8 @@ def get_github_repos(url, headers, gh_org): res = requests.request('GET', url=req_url, headers=headers) if res.json(): for repo in res.json(): - repositories.append(repo) + if repo['archived'] == False: + repositories.append(repo) i+=1 continue else: @@ -254,6 +279,7 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): if res_sta.json(): if len(res_sta.json()['check_runs']) != 0: if res_sta.json()['check_runs'][0]['conclusion'] == 'failure': + zuul_url = res_sta.json()['check_runs'][0]['details_url'] o = FailedPR( host='github', url=pull['html_url'], @@ -261,13 +287,13 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): repo=repo['name'], pullrequest=pull['title'], status=res_sta.json()['check_runs'][0]['conclusion'], - zuul_url=(res_sta.json()['check_runs'] - [0]['details_url']), + zuul_url=zuul_url, created_at=pull['created_at'], updated_at=(res_sta.json()['check_runs'] [0]['completed_at']), error=1000 ) + o = add_builds_to_obj(obj=o, url=zuul_url, tenant='eco') failed_commits.append(o) else: o = FailedPR( @@ -286,8 +312,6 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): print("An error has occured: " + str(e)) print("The request status is: " + str(res_sta.status_code) + " | " + str(res_sta.reason)) - print(json.dumps(res_sta.json())) - exit() return failed_commits -- 2.34.1 From 14a8cfa18c259d2d5e48c66df0383b6bcc7e5911 Mon Sep 17 00:00:00 2001 From: tischrei Date: Wed, 28 Sep 2022 13:08:11 +0000 Subject: [PATCH 16/16] add yaml option --- tools/attention_list.py | 136 +++++++++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 44 deletions(-) diff --git a/tools/attention_list.py b/tools/attention_list.py index 8152e80a..027295b1 100644 --- a/tools/attention_list.py +++ b/tools/attention_list.py @@ -22,6 +22,8 @@ import logging import requests import json import re +import os +import yaml class FailedPR: @@ -59,7 +61,7 @@ def get_args(): parser = argparse.ArgumentParser( description='Bootstrap repositories.') parser.add_argument( - '--gh-orgs', + '--github-orgs', nargs='+', default=['opentelekomcloud-docs'], help='One or more GitHub organizations to be queried for failed Pull ' @@ -75,7 +77,7 @@ def get_args(): 'Default: [docs]' ) parser.add_argument( - '--gh-token', + '--github-token', help='API Token for GitHub.' ) parser.add_argument( @@ -94,7 +96,7 @@ def get_args(): 'Default: https://gitea.eco.tsi-dev.otc-service.com/api/v1/' ) parser.add_argument( - '--gh-url', + '--github-url', default='https://api.github.com/', help='GitHub Base URL for API request.\n' 'Default: https://api.github.com/' @@ -106,8 +108,14 @@ def get_args(): help='Git hoster to be queried for failed Pull Requests.\n' 'Default: [github, gitea].' ) + parser.add_argument( + '--yaml', + action='store_true', + help='Set yaml output format instead of json.' + ) return parser.parse_args() + def add_builds_to_obj(obj, url, tenant): """ This method trys to find all build jobs under a Zuul buildset. @@ -132,6 +140,7 @@ def add_builds_to_obj(obj, url, tenant): obj.jobs = jobs return obj + def get_gitea_repos(url, headers, gitea_org): """ Get all Repositories of one Gitea orgainzation @@ -157,10 +166,11 @@ def get_gitea_repos(url, headers, gitea_org): except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res.status_code) + - " | " + str(res.reason)) + " | " + str(res.reason)) break return repositories + def get_gitea_prs(url, headers, gitea_org, repo): """ Collect all Pull Requests of a Gitea Repository @@ -169,7 +179,7 @@ def get_gitea_prs(url, headers, gitea_org, repo): try: req_url = (url + 'repos/' + gitea_org + '/' - + repo + '/pulls?state=open') + + repo + '/pulls?state=open') res = requests.request('GET', url=req_url, headers=headers) if res.json(): for pr in res.json(): @@ -177,18 +187,19 @@ def get_gitea_prs(url, headers, gitea_org, repo): except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res.status_code) + - " | " + str(res.reason)) + " | " + str(res.reason)) exit() return pullrequests -def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): + +def get_gitea_failed_commits(pull, url, gitea_org, repo, headers): """ Collect all failed gitea commits of one repository. """ failed_commits = [] try: req_url = (url + 'repos/' + gitea_org + '/' + repo['name'] + - '/commits/' + pull['head']['ref'] + '/statuses?limit=1') + '/commits/' + pull['head']['ref'] + '/statuses?limit=1') res_sta = requests.request('GET', url=req_url, headers=headers) if res_sta.json(): if res_sta.json()[0]['status'] == 'failure': @@ -211,10 +222,11 @@ def get_failed_gitea_commits(pull, url, gitea_org, repo, headers): except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res_sta.status_code) + - " | " + str(res_sta.reason)) + " | " + str(res_sta.reason)) return failed_commits -def get_github_repos(url, headers, gh_org): + +def get_github_repos(url, headers, github_org): """ Get all repositories of one GitHub organization """ @@ -223,24 +235,25 @@ def get_github_repos(url, headers, gh_org): while True: try: - req_url = url + 'orgs/' + gh_org + '/repos?page=' + str(i) + req_url = url + 'orgs/' + github_org + '/repos?page=' + str(i) res = requests.request('GET', url=req_url, headers=headers) if res.json(): for repo in res.json(): - if repo['archived'] == False: + if repo['archived'] is False: repositories.append(repo) - i+=1 + i += 1 continue else: break except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res.status_code) + - " | " + str(res.reason)) + " | " + str(res.reason)) break return repositories -def get_github_prs(url, headers, gh_org, repo): + +def get_github_prs(url, headers, github_org, repo): """ Get all Pull Requests of one GitHub repository """ @@ -249,32 +262,32 @@ def get_github_prs(url, headers, gh_org, repo): while True: try: - req_url = (url + 'repos/' + gh_org + '/' + repo + - '/pulls?state=open&page=' + str(i)) + req_url = (url + 'repos/' + github_org + '/' + repo + + '/pulls?state=open&page=' + str(i)) res = requests.request('GET', url=req_url, headers=headers) if res.json(): for pr in res.json(): pullrequests.append(pr) - i+=1 + i += 1 continue else: break except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res.status_code) + - " | " + str(res.reason)) + " | " + str(res.reason)) break return pullrequests -def get_failed_gh_commits(pull, url, gh_org, repo, headers): +def get_github_failed_commits(pull, url, github_org, repo, headers): """ Collect all Failed Pull Requests of one GitHub repository """ failed_commits = [] try: - req_url = (url + 'repos/' + gh_org + '/' + repo['name'] + - '/commits/' + pull['head']['sha'] + '/check-runs') + req_url = (url + 'repos/' + github_org + '/' + repo['name'] + + '/commits/' + pull['head']['sha'] + '/check-runs') res_sta = requests.request('GET', url=req_url, headers=headers) if res_sta.json(): if len(res_sta.json()['check_runs']) != 0: @@ -283,7 +296,7 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): o = FailedPR( host='github', url=pull['html_url'], - org=gh_org, + org=github_org, repo=repo['name'], pullrequest=pull['title'], status=res_sta.json()['check_runs'][0]['conclusion'], @@ -299,7 +312,7 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): o = FailedPR( host='github', url=pull['html_url'], - org=gh_org, + org=github_org, repo=repo['name'], pullrequest=pull['title'], created_at=pull['created_at'], @@ -311,11 +324,11 @@ def get_failed_gh_commits(pull, url, gh_org, repo, headers): except Exception as e: print("An error has occured: " + str(e)) print("The request status is: " + str(res_sta.status_code) + - " | " + str(res_sta.reason)) + " | " + str(res_sta.reason)) return failed_commits -def create_json_result(failed_commits): +def create_result(failed_commits): """ Create Result """ @@ -331,7 +344,32 @@ def create_json_result(failed_commits): else: result['meta']['count'] = 0 - return json.dumps(result) + return result + + +def get_token(hoster, args): + token = '' + if hoster == 'github': + if args.github_token: + token = args.github_token + elif os.getenv('GITHUB_TOKEN'): + token = os.getenv('GITHUB_TOKEN') + else: + raise Exception( + 'Please, provide GitHub Token as console argument or \n' + 'as environment variable GITHUB_TOKEN') + elif hoster == 'gitea': + if args.gitea_token: + token = args.gitea_token + elif os.getenv('GITEA_TOKEN'): + token = os.getenv('GITEA_TOKEN') + else: + raise Exception( + 'Please, provide GitHub Token as console argument or \n' + 'as environment variable GITEA_TOKEN') + else: + raise ValueError('No supported Git hoster provided.') + return token def main(): @@ -346,11 +384,13 @@ def main(): headers = {} headers['accept'] = 'application/json' - if not args.gitea_token: - raise Exception('Please, provide Gitea Token as argument.') + headers['Authorization'] = 'token ' + get_token( + hoster=h, + args=args + ) + if not args.gitea_url: raise Exception('Please, provide Gitea URL as argument.') - headers['Authorization'] = 'token ' + args.gitea_token for org in args.gitea_orgs: repos = get_gitea_repos( @@ -367,7 +407,7 @@ def main(): ) if pulls: for pull in pulls: - commits = get_failed_gitea_commits( + commits = get_gitea_failed_commits( pull=pull, url=args.gitea_url, gitea_org=org, @@ -377,41 +417,49 @@ def main(): failed_commits.extend(commits) elif h == 'github': - if not args.gh_url: - raise ValueError('Parameter --gh-url not found.') - url = args.gh_url + if not args.github_url: + raise ValueError('Parameter --github-url not found.') + url = args.github_url headers = {} headers['accept'] = 'application/json' - if not args.gh_token: - raise Exception('Please, provide GitHub Token as argument.') - headers['Authorization'] = 'Bearer ' + args.gh_token + headers['Authorization'] = 'Bearer ' + get_token( + hoster=h, + args=args + ) - for org in args.gh_orgs: + for org in args.github_orgs: repos = get_github_repos( url=url, headers=headers, - gh_org=org + github_org=org ) for repo in repos: pulls = get_github_prs( url=url, headers=headers, - gh_org=org, + github_org=org, repo=repo['name'] ) if pulls: for pull in pulls: - commits = get_failed_gh_commits( + commits = get_github_failed_commits( pull=pull, url=url, - gh_org=org, + github_org=org, repo=repo, headers=headers ) failed_commits.extend(commits) + else: + raise ValueError("No supported hoster found.") - print(create_json_result( - failed_commits=failed_commits)) + result = create_result(failed_commits=failed_commits) + if args.yaml: + result = yaml.dump(result) + else: + result = json.dumps(result) + + print(result) if __name__ == '__main__': -- 2.34.1