diff --git a/config.json b/config.json new file mode 100644 index 0000000..7ea2fbc --- /dev/null +++ b/config.json @@ -0,0 +1,57 @@ +{ + "watchdog_file": "hc_spider.fifo", + "timer_runtime": 0, + "log_dir": "log", + "logging_interval": 10, + "otc_base_url": "https://docs.otc.t-systems.com", + "workers": 0, + "starting_point": "https://docs.otc.t-systems.com/", + "blacklisted_domains": [ + "pypi.org", + "python-otcextensions.readthedocs.io", + "ubuntu.com", + "grafana.com", + "kubernetes.io", + "golang.org", + "cwiki.apache.org", + "registry.terraform.io", + "addons.mozilla.org", + "chrome.google.com", + "learn.hashicorp.com", + "www.conjur.org", + "docs.openstack.org", + "www.vaultproject.io", + "www.hashicorp.com", + "github.com", + "kafka.apache.org" + ], + "blacklisted_urls": [ + "https://docs.otc.t-systems.com/en-us/api/apiug/apig-en-api-180328001.html?tag=API%20Documents", + "https://docs.otc.t-systems.com/en-us/api/apiug/apig-en-api-180328001.html?tag=API Documents", + "https://docs.otc.t-systems.com/en-us/api/apiug/apig-en-api-180328001.html?tag=API%20Documents%20", + "https://enterprise-dashboard.obs.eu-de.otc.t-systems.com/OTC%20Enterprise%20Dashboard%20Usermanual.pdf", + "https://open-telekom-cloud.com/", + "https://curl.haxx.se/", + "https://www.t-systems.com/de/en/disclaimer-of-liability", + "https://www.t-systems.com/de/en/imprint", + "https://cloud.telekom.de/en/contact/", + "https://www.t-systems.com/de/en/data-protection", + "https://open-telekom-cloud.com/service-description", + "https://nvd.nist.gov/", + "https://community.open-telekom-cloud.com/community", + "https://docs.otc.t-systems.com", + "https://console.otc.t-systems.com", + "https://status.otc-service.com", + "https://docs.otc.t-systems.com/status-dashboard/umn/_static/OTC_Status_Dashboard_User_Guide.pdf", + "https://enterprise-dashboard.otc-service.com", + "https://open-telekom-cloud.com/de/support/release-notes", + "https://docs.otc.t-systems.com/releasenotes/python-otcextensions", + "https://docs.otc.t-systems.com/releasenotes/terraform-provider-opentelekomcloud", + "https://docs.otc.t-systems.com/releasenotes/otcdocstheme", + "https://open-telekom-cloud.com/en/contact", + "https://open-telekom-cloud.com/en/disclaimer-of-liability", + "https://open-telekom-cloud.com/en/data-protection", + "https://open-telekom-cloud.com/en/imprint", + "http://www.tpc.org/tpcds/" + ] +} \ No newline at end of file