infra/otc-metadata-rework
14
0
Fork 5
Code Issues 5 Pull Requests 2 Actions Packages Releases 1 Wiki Activity

Added logging for updated files #81

Open
sgode wants to merge 3 commits from logging into main
pull from: logging
merge into: infra:main
Conversation 0 Commits 3 Files Changed 1 +27 -5
1 changed files with 27 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
tools/generate_meta.py
View File

@ -86,8 +86,9 @@ def generate_description_with_llm(text, service_title, llm_api_url, model_name,
"""Generate a meta description using the llama.cpp /completion endpoint with up to 3 retries.""" """Generate a meta description using the llama.cpp /completion endpoint with up to 3 retries."""
content_preview = text[:2000].replace("\n", " ") content_preview = text[:2000].replace("\n", " ")
prompt = ( prompt = (
f"Generate a meta description (40-160 chars) for: {service_title}." f"Generate a meta description (40-160 chars) for the T Cloud Public service: {service_title}."
f"This is a service from the cloud provider called 'T Cloud Public', do not mention other Cloud Providers or services from them." f"This is a service from the cloud provider called 'T Cloud Public', do NOT mention other Cloud Providers or services from them."
f"NEVER mention AWS, Azure, Huawei, Alibaba, GCP and similiar cloud providers."
f"Content preview: {content_preview}." f"Content preview: {content_preview}."
f"Output ONLY the description text, nothing else." f"Output ONLY the description text, nothing else."
) )
@ -160,8 +161,9 @@ def generate_keywords_with_llm(text, service_title, llm_api_url, model_name, api
"""Generate keywords using the llama.cpp /completion endpoint with up to 3 retries.""" """Generate keywords using the llama.cpp /completion endpoint with up to 3 retries."""
content_preview = text[:2000].replace("\n", " ") content_preview = text[:2000].replace("\n", " ")
prompt = ( prompt = (
f"Generate up to 5 keywords (comma-separated) for: {service_title}. " f"Generate up to 5 keywords (comma-separated) for the T Cloud Public service: {service_title}. "
f"This is a service from the cloud provider called 'T Cloud Public', do not mention other Cloud Providers or services from them." f"This is a service from the cloud provider called 'T Cloud Public', do NOT mention other Cloud Providers or services from them."
f"NEVER mention AWS, Azure, Huawei, Alibaba, GCP and similiar cloud providers."
f"Content preview: {content_preview}. " f"Content preview: {content_preview}. "
f"Output ONLY comma-separated keywords, nothing else." f"Output ONLY comma-separated keywords, nothing else."
) )
@ -256,6 +258,7 @@ def process_service(args, service):
repo_dir = None repo_dir = None
git_repo = None git_repo = None
error_list = [] error_list = []
files_updated = 0
repo = None repo = None
for r in service["repositories"]: for r in service["repositories"]:
@ -352,6 +355,7 @@ def process_service(args, service):
if add_sphinx_metadata(rst_file, description, keywords): if add_sphinx_metadata(rst_file, description, keywords):
updated_count += 1 updated_count += 1
files_updated += 1
logging.info(f"Added meta description and keywords to {rst_file}") logging.info(f"Added meta description and keywords to {rst_file}")
else: else:
processed_count += 1 processed_count += 1
@ -387,6 +391,8 @@ def process_service(args, service):
logging.info(f"Processed {processed_count} files, updated {updated_count} files") logging.info(f"Processed {processed_count} files, updated {updated_count} files")
return files_updated
def main(): def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
@ -453,8 +459,24 @@ def main():
else: else:
services = data.services_with_repos() services = data.services_with_repos()
total_files_updated = 0
services_with_updates = []
for service in services: for service in services:
process_service(args, service) files_updated = process_service(args, service)
if files_updated > 0:
total_files_updated += files_updated
services_with_updates.append((service['service_title'], files_updated))
print("\n" + "=" * 60)
print("SUMMARY")
print("=" * 60)
print(f"Total files with metadata added: {total_files_updated}")
print(f"Services with updates: {len(services_with_updates)}")
print("\nServices with metadata added:")
for service_title, count in services_with_updates:
print(f" - {service_title}: {count} file(s)")
print("=" * 60)
if __name__ == "__main__": if __name__ == "__main__":