Merge pull request 'lots of updates' (#1) from init into main

Reviewed-on: #1

.gitignore

@@ -159,4 +159,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
-
+playbooks/certs
+certs
+inventory/sensitive

README.rst

@@ -0,0 +1,50 @@
+=========================================
+OpenTelekomCloud SCS System Configuration
+=========================================
+
+This is the machinery that drives the configuration, testing, continuous
+integration and deployment of services provided by the OpenTelekomCloud
+project. It heavily copies OpenDev configuration approach with some extensions
+and deviations.
+
+Services are driven by Ansible playbooks and associated roles stored here. If
+you are interested in the configuration of a particular service, starting at
+``playbooks/service-<name>.yaml`` will show you how it is configured.
+
+Most services are deployed via containers; many of them are built or customised
+in this repository; see ``docker/``.
+
+Bootstrap
+=========
+
+Bootstraping new installation is connected with usual
+chicken-egg problem. Generally having system up and running it
+is required to maintain certain secrets. But providing those
+secrets requires infrastructure to be up and running. Addressing
+this requres certain steps.
+
+TLS Certificates
+----------------
+
+Most systems require valid TLS certificates. Initial bootstraping also requires valid TLS certificates. System that require those will typically support providing of initial certificates through inventory variables.
+
+Vault
+-----
+
+Managing secrets securely is possible in few different ways.
+Ansible vault is a good tool, but it is complex to manage unseal
+and to implement rotations (of both vault password as well as
+secrets inside the vault).
+HashiCorp Vault is in that sense a much more flexible system that also provides support for infrastructure based authorization.
+
+Deploying Vault on the other side is also requiring SSL certificates. Since during bootstraping it is most likely not possible to rely on the `playbooks/acme-certs.yaml` since it requires bootstrapped bridge host first it is required to provide initial valid certificates through host variables (`vault_tls_cert_content` and `vault_tls_key_content`). It makes sense not to commit those variables under the git and only provide them during the bootstraping phase.
+
+Bootstraping Vault therefore requires following steps
+
+1. Login to the host having access to all nodes which will host HashiCorp vault
+
+2. Checkout this repository and ensure
+   `inventory/service/hosts.yaml` contain proper IP addresses as
+   well as those hosts are member of vault group as `inventory/service/all.yaml`
+
+3. execute `ansible-playbook playbooks/service-vault.yaml` playbook.

bindep.txt

@@ -0,0 +1,5 @@
+libffi-dev [platform:dpkg]
+libffi-devel [platform:rpm]
+libssl-dev [platform:dpkg]
+openssl-devel [platform:rpm]
+graphviz [doc]

inventory/local_certs/group_vars/all.yaml

@@ -0,0 +1 @@
+certs_path: "../certs"

inventory/local_certs/group_vars/vault.yaml

@@ -0,0 +1,2 @@
+vault_tls_cert_content: "{{ lookup('ansible.builtin.file', certs_path + '/' + vault_cert + '-fullchain.crt') | default(omit) }}"
+vault_tls_key_content: "{{ lookup('ansible.builtin.file', certs_path + '/' + vault_cert + '.pem') }}"

inventory/service/all.yaml

@@ -0,0 +1,60 @@
+---
+all:
+  vars:
+    # ansible_ssh_user: ubuntu
+    ansible_ssh_private_key_file: ~/id_rsa.scs-muneeb-general
+      #/root/.ssh/id_rsa_scs
+  hosts:
+    localhost:
+      ansible_connection: local
+    bastion.scs.otc-service.com:
+      ansible_host: 10.0.20.232
+      ansible_user: automation
+      public_v4: 10.0.20.232
+    vault1.scs.otc-service.com:
+      ansible_host: 10.10.0.24
+      public_v4: 10.10.0.24
+    vault2.scs.otc-service.com:
+      ansible_host: 10.10.0.223
+      public_v4: 10.10.0.223
+    vault3.scs.otc-service.com:
+      ansible_host: 10.10.0.234
+      public_v4: 10.10.0.234
+        #    gitea1.scs.otc-service.com:
+        #      ansible_host: 10.10.0.107
+        #      ansible_user: automation
+        #      public_v4: 10.10.0.107
+        #      host_keys:
+        #        - 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIKrZwdNgGFNSozidYBIyFTla9Ho6ZksBQZja3jBhtnMabm2eUk0ITvaIvAhhhXGk2XeiRzvWpc/WtroIMLm+w0='
+  children:
+    bastion:
+      hosts:
+        bastion*.scs.otc-service.com:
+
+    ssl_certs:
+      hosts:
+        bridge.scs.otc-service.com:
+        vault1.scs.otc-service.com:
+        vault2.scs.otc-service.com:
+        vault3.scs.otc-service.com:
+          #        gitea1.scs.otc-service.com:
+
+    k8s-controller:
+      hosts:
+        bridge.scs.otc-service.com:
+
+    vault:
+      hosts:
+        vault1.scs.otc-service.com:
+        vault2.scs.otc-service.com:
+        vault3.scs.otc-service.com:
+
+    vault-controller:
+      hosts:
+        localhost:
+
+    gitea:
+      hosts:
+        gitea1.scs.otc-service.com:
+
+    disabled: []

inventory/service/group_vars/all.yaml

@@ -0,0 +1,12 @@
+vault_image_stable: quay.io/opentelekomcloud/vault:change_668_latest
+vault_image_latest: quay.io/opentelekomcloud/vault:change_668_latest
+
+distro_lookup_path:
+  - "{{ ansible_facts.distribution }}.{{ ansible_facts.lsb.codename|default() }}.{{ ansible_facts.architecture }}.yaml"
+  - "{{ ansible_facts.distribution }}.{{ ansible_facts.lsb.codename|default() }}.yaml"
+  - "{{ ansible_facts.distribution }}.{{ ansible_facts.architecture }}.yaml"
+  - "{{ ansible_facts.distribution }}.yaml"
+  - "{{ ansible_facts.os_family }}.yaml"
+  - "default.yaml"
+
+

inventory/service/group_vars/bastion.yaml

@@ -0,0 +1,8 @@
+bastion_key_exclusive: false
+
+kube_config_template: clouds/bridge_kube_config.yaml.j2
+extra_users:
+  - zuul
+
+cloud_launcher_profiles: []
+cloud_launcher_clouds: []

inventory/service/group_vars/cloud-launcher.yaml

@@ -0,0 +1,2 @@
+ansible_roles:
+  - cloud-launcher

inventory/service/group_vars/gitea.yaml

@@ -0,0 +1,30 @@
+gitea_version: "1.18.5"
+gitea_checksum: "sha256:4766ad9310bd39d50676f8199563292ae0bab3a1922b461ece0feb4611e867f2"
+
+gitea_domain: "gitea.eco.tsi-dev.otc-service.com"
+gitea_app_name: "Open Telekom Cloud: git"
+gitea_root_url: "https://gitea.eco.tsi-dev.otc-service.com"
+gitea_http_port: 443
+gitea_packages_enable: true
+
+fail2ban_filters:
+  - content: |
+      # gitea.conf
+      [Init]
+      datepattern = ^%%Y/%%m/%%d %%H:%%M:%%S
+
+      [Definition]
+      failregex =  .*(Failed authentication attempt|invalid credentials|Attempted access of unknown user).* from <HOST>
+      ignoreregex =
+    dest: "/etc/fail2ban/filter.d/gitea.conf"
+
+fail2ban_jails:
+  - content: |
+      [gitea]
+      enabled = true
+      filter = gitea
+      logpath = /var/lib/gitea/log/gitea.log
+      maxretry = 10
+      findtime = 3600
+      bantime = 900
+    dest: "/etc/fail2ban/jail.d/gitea.conf"

inventory/service/group_vars/k8s-controller.yaml

@@ -0,0 +1,28 @@
+---
+helm_chart_instances:
+  otcci_cert-manager:
+    context: otcci
+    repo_url: https://charts.jetstack.io
+    repo_name: jetstack
+    name: cert-manager
+    ref: jetstack/cert-manager
+    version: v1.6.1
+    namespace: cert-manager
+    values_template: "templates/charts/cert-manager/cert-manager-values.yaml.j2"
+    post_config_template: "templates/charts/cert-manager/cert-manager-post-config.yaml.j2"
+  otcci_nginx-ingress:
+    context: otcci
+    repo_url: https://kubernetes.github.io/ingress-nginx
+    repo_name: ingress-nginx
+    name: ingress-nginx
+    ref: ingress-nginx/ingress-nginx
+    version: 4.1.0
+    namespace: default
+    values_template: "templates/charts/ingress-nginx/ingress-nginx-values.yaml.j2"
+    is_default: true
+    config_entries:
+      use-gzip: true
+      compute-full-forwarded-for: true
+      use-forwarded-headers: true
+    elb_id: "3d926b98-97ec-4060-be79-ac67c82298e7"
+    elb_eip: "80.158.57.224"

inventory/service/group_vars/vault.yaml

@@ -0,0 +1,19 @@
+# Vault settings
+vault_plugins:
+  #  - url: "https://github.com/opentelekomcloud-infra/vault-plugin-secrets-github/releases/download/v1.2.1/vault-plugin-secrets-github_1.2.1_linux_amd64.zip"
+  #    sha256: "9acd271a264a48cb8dfac055bb9849b3938fe8afbc794a2d81d14be1357cbcf5"
+  #    name: "vault-plugin-secrets-github"
+  #    type: "secret"
+  #    paths:
+  #      - "github"
+  #      - "github_otcbot"
+  #      - "github_zuul"
+  #  - url: "https://github.com/opentelekomcloud/vault-plugin-secrets-openstack/releases/download/v1.3.0/vault-plugin-secrets-openstack_1.3.0_linux_amd64.tar.gz"
+  #    sha256: "2f48d3011a0cc0ce4726e889f5d4103446eb820cdcc0ecb89deb03757e42568e"
+  #    name: "vault-plugin-secrets-openstack"
+  #    type: "secret"
+  #    paths:
+  #      - "openstack"
+certs_path: "../certs"
+vault_tls_cert_content: "{{ lookup('ansible.builtin.file', certs_path + '/' + vault_cert + '-fullchain.crt') | default(omit) }}"
+vault_tls_key_content: "{{ lookup('ansible.builtin.file', certs_path + '/' + vault_cert + '.pem') }}"

inventory/service/host_vars/bastion.scs.otc-service.com.yaml

@@ -0,0 +1,5 @@
+firewalld_extra_ports_enable: []
+
+# Allow tcp and agent forwarding on the jump host. Aligned with DT 3.04-19/20
+ssh_allow_tcp_forwarding: true
+ssh_allow_agent_forwarding: true

inventory/service/host_vars/vault1.scs.otc-service.com.yaml

@@ -0,0 +1,13 @@
+---
+ssl_certs:
+  vault:
+    - "vault1.scs.otc-service.com"
+vault_cert: "vault1"
+vault_node_id: "vault-eu-de-01-Instance-01"
+
+vault_cluster_nodes:
+  - vault1.scs.otc-service.com
+  - vault2.scs.otc-service.com
+  - vault3.scs.otc-service.com
+
+firewalld_extra_ports_enable: ['8200/tcp', '8201/tcp']

inventory/service/host_vars/vault2.scs.otc-service.com.yaml

@@ -0,0 +1,13 @@
+---
+ssl_certs:
+  vault2:
+    - "vault2.scs.otc-service.com"
+vault_cert: "vault2"
+vault_node_id: "vault-eu-de-02-Instance-01"
+
+vault_cluster_nodes:
+  - vault1.scs.otc-service.com
+  - vault2.scs.otc-service.com
+  - vault3.scs.otc-service.com
+
+firewalld_extra_ports_enable: ['8200/tcp', '8201/tcp']

inventory/service/host_vars/vault3.scs.otc-service.com.yaml

@@ -0,0 +1,13 @@
+---
+ssl_certs:
+  vault:
+    - "vault3.scs.otc-service.com"
+vault_cert: "vault3"
+vault_node_id: "vault-eu-de-03-Instance-01"
+
+vault_cluster_nodes:
+  - vault1.scs.otc-service.com
+  - vault2.scs.otc-service.com
+  - vault3.scs.otc-service.com
+
+firewalld_extra_ports_enable: ['8200/tcp', '8201/tcp']

kubernetes/zuul/README.md

@@ -0,0 +1,72 @@
+# Kustomize stack for installing Zuul
+
+This folder contains Kubernetes manifests processed by Kustomize application in
+order to generate final set of manifests for installing Zuul into the
+Kubernetes.
+
+## Components
+
+Whole installation is split into individual components, so that it is possible
+to configure what to use in a specific installation:
+
+### ca
+
+Zuul requires Zookeeper in HA mode with TLS enabled to function. It is possible
+to handle TLS outside of the cluster, but it is also possible to rely on
+cert-manager capability of having own CA authority and provide certificates as
+requested. At the moment this is set as a hard dependency in the remaining
+components, but it would be relatively easy to make it really optional
+component.
+
+### Zookeeper
+
+This represents a Zookeeper cluster installation. No crazy stuff, pretty
+straigt forward
+
+### zuul-scheduler
+
+Zuul scheduler
+
+### zuul-executor
+
+Zuul executor
+
+### zuul-merger
+
+Optional zuul-merger
+
+### zuul-web
+
+Zuul web frontend
+
+### nodepool-launcher
+
+Launcher for VMs or pods
+
+### nodepool-builder
+
+Optional builder for VM images. At the moment it is not possible to build all
+types of images inside of Kubernetes, since running podman under docker in K8
+is not working smoothly on every installation
+
+## Layers
+
+- `base` layer is representing absolutely minimal installaiton. In the
+  kustomization.yaml there is a link to zuul-config repository which must
+  contain `nodepool/nodepool.yaml` - nodepool config and `zuul/main.yaml` -
+  tenants info.  This link is given by `zuul_instance_config` configmap with
+  ZUUL_CONFIG_REPO=https://gitea.eco.tsi-dev.otc-service.com/scs/zuul-config.git
+
+- `zuul_ci` - zuul.otc-service.com installation
+
+## Versions
+
+Zookeeper version is controlled through
+`components/zookeeper/kustomization.yaml`
+
+Zuul version by default is pointing to the latest version in docker registry
+and it is expected that every overlay is setting desired version.
+
+Proper overlays are also relying on HashiCorp Vault for providing installation
+secrets. Vault agent version is controlled i.e. in the overlay itself with
+variable pointing to the vault installation in the overlay patch.

kubernetes/zuul/overlays/scs/configs/openstack/clouds.yaml.hcl

@@ -0,0 +1,16 @@
+---
+# Nodepool openstacksdk configuration
+#
+# This file is deployed to nodepool launcher and builder hosts
+# and is used there to authenticate nodepool operations to clouds.
+# This file only contains projects we are launching test nodes in, and
+# the naming should correspond that used in nodepool configuration
+# files.
+#
+# Generated automatically, please do not edit directly!
+cache:
+  expiration:
+    server: 5
+    port: 5
+    floating-ip: 5
+clouds: []

kubernetes/zuul/overlays/scs/configs/vault-agent/config-nodepool.hcl

@@ -0,0 +1,37 @@
+pid_file = "/home/vault/.pid"
+vault {
+  address = "https://vault-lb.scs.otc-service.com:8200"
+  retry {
+    num_retries = 5
+  }
+}
+auto_auth {
+    method {
+        "mount_path" = "auth/kubernetes_scs"
+        "config" = {
+          "role" = "zuul"
+        }
+        "type" = "kubernetes"
+      }
+    sink "file" {
+        config = {
+            path = "/home/vault/.token"
+        }
+    }
+}
+
+api_proxy {
+  use_auto_auth_token = "true"
+  enforce_consistency = "always"
+}
+
+listener "tcp" {
+  address = "127.0.0.1:8100"
+  tls_disable = true
+}
+
+template {
+  destination = "/vault/secrets/openstack/clouds.yaml"
+  source = "/vault/custom/clouds.yaml.hcl"
+  perms = "0640"
+}

kubernetes/zuul/overlays/scs/configs/vault-agent/config-zuul.hcl

@@ -0,0 +1,62 @@
+pid_file = "/home/vault/.pid"
+vault {
+  address = "https://vault-lb.scs.otc-service.com:8200"
+  retry {
+    num_retries = 5
+  }
+}
+
+auto_auth {
+    method {
+        "mount_path" = "auth/kubernetes_scs"
+        "config" = {
+          "role" = "zuul"
+        }
+        "type" = "kubernetes"
+      }
+    sink "file" {
+        config = {
+            path = "/home/vault/.token"
+        }
+    }
+}
+
+api_proxy {
+  use_auto_auth_token = "true"
+  enforce_consistency = "always"
+}
+
+listener "tcp" {
+  address = "127.0.0.1:8100"
+  tls_disable = true
+}
+
+template {
+  destination = "/vault/secrets/connections/github.key"
+  contents = <<EOT
+{{- with secret "secret/zuul/connections/github" }}{{ .Data.data.app_key }}{{ end }}
+EOT
+  perms = "0600"
+}
+
+template {
+  destination = "/vault/secrets/connections/gitea.key"
+  contents = <<EOT
+{{ with secret "secret/zuul/connections/gitea" }}{{ .Data.data.ssh_key }}{{ end }}
+EOT
+  perms = "0600"
+}
+
+template {
+  destination = "/vault/secrets/zuul.conf"
+  source = "/vault/custom/zuul.conf.hcl"
+  perms = "0644"
+  # exec = { command = "sh -c '{ if [ -f /secrets/config.check ]; then kubectl -n zuul-ci rollout restart statefulset zuul-executor; else touch /secrets/config.check; fi }'", timeout = "30s" }
+}
+template {
+  destination = "/vault/secrets/sshkey"
+  contents = <<EOT
+{{- with secret "secret/zuul/sshkey" }}{{ .Data.data.private_key }}{{ end }}
+EOT
+  perms = "0600"
+}

kubernetes/zuul/overlays/scs/configs/zuul.conf.hcl

@@ -0,0 +1,78 @@
+[zookeeper]
+hosts=zookeeper.zuul-ci.svc.cluster.local:2281
+tls_cert=/tls/client/tls.crt
+tls_key=/tls/client/tls.key
+tls_ca=/tls/client/ca.crt
+session_timeout=40
+
+[scheduler]
+tenant_config=/etc/zuul-config/zuul/main.yaml
+state_dir=/var/lib/zuul
+relative_priority=true
+prometheus_port=9091
+
+[web]
+listen_address=0.0.0.0
+port=9000
+status_url=https://zuul.scs.otc-service.com
+root=https://zuul.scs.otc-service.com
+prometheus_port=9091
+
+[fingergw]
+port=9079
+user=zuul
+
+[keystore]
+{{- with secret "secret/zuul/keystore_password" }}
+password={{ .Data.data.password }}
+{{- end }}
+
+[merger]
+git_dir=/var/lib/zuul/git
+git_timeout=600
+git_user_email=zuul@zuul.scs.otc-service.com
+git_user_name=OTC SCS Zuul
+prometheus_port=9091
+
+[executor]
+manage_ansible=true
+ansible_root=/var/lib/zuul/managed_ansible
+private_key_file=/etc/zuul/sshkey
+disk_limit_per_job=2000
+max_starting_builds=5
+trusted_ro_paths=/var/run/zuul/trusted-ro
+variables=/var/run/zuul/vars/site-vars.yaml
+prometheus_port=9091
+
+[database]
+{{- with secret "secret/zuul/db" }}
+dburi=postgresql://{{ .Data.data.username }}:{{ .Data.data.password }}@10.20.0.21:8635/zuul?sslmode=require
+{{- end }}
+
+# base zuul jobs
+[connection "opendev"]
+name=opendev
+driver=git
+baseurl=https://opendev.org
+
+# zuul-config repository with tenants config
+[connection "github"]
+name=github
+driver=github
+{{- with secret "secret/zuul/connections/github" }}
+webhook_token={{ .Data.data.webhook_token }}
+app_id={{ .Data.data.app_id }}
+{{- end }}
+app_key=/etc/zuul/connections/github.key
+
+[connection "gitea"]
+name=gitea
+driver=gitea
+baseurl=https://gitea.eco.tsi-dev.otc-service.com
+server=gitea.eco.tsi-dev.otc-service.com
+cloneurl=ssh://git@gitea.eco.tsi-dev.otc-service.com:2222
+{{- with secret "secret/zuul/connections/gitea" }}
+api_token={{ .Data.data.api_token }}
+webhook_secret={{ .Data.data.webhook }}
+{{- end }}
+sshkey=/etc/zuul/connections/gitea.key

kubernetes/zuul/overlays/scs/crb.yaml

@@ -0,0 +1,12 @@
+apiVersion: "rbac.authorization.k8s.io/v1"
+kind: "ClusterRoleBinding"
+metadata:
+  name: zuul-vault-crb
+roleRef:
+  apiGroup: "rbac.authorization.k8s.io"
+  kind: "ClusterRole"
+  name: "system:auth-delegator"
+subjects:
+  - kind: "ServiceAccount"
+    name: "zuul"
+    namespace: "zuul-ci"

playbooks/acme-certs-local.yaml

@@ -0,0 +1,56 @@
+---
+# This playbook is intended to be used i.e. during bootstraping
+# of the system to generate required TLS certs using Acme
+# (LetsEncrypt) on localhost.
+# It expects following variables:
+# - :dns_cloud: name of the clouds.yaml entry on the localhost
+#       which is hosting target DNS zone
+#
+# - :certs_path: path to the folder where certificates would be
+#       places
+#
+# - :ssl_certs: a dictionary for the requested certificates
+#       key: cert name (file name) to be used
+#       value: list of DNS names
+- hosts: localhost
+  # become: true
+  vars_prompt:
+
+    - name: certs_path
+      prompt: Path to the certs folder
+      default: "certs"
+
+  vars:
+    ssl_certs:
+      vault1:
+        - vault1.scs.otc-service.com
+        - vault-lb.scs.otc-service.com
+      vault2:
+        - vault2.scs.otc-service.com
+        - vault-lb.scs.otc-service.com
+      vault3:
+        - vault3.scs.otc-service.com
+        - vault-lb.scs.otc-service.com
+
+  tasks:
+    - name: Generate CSRs
+      ansible.builtin.include_role:
+        name: acme_request_certs
+
+    - name: Add localhost into the ssl_certs group as expected by the role
+      ansible.builtin.add_host:
+        name: localhost
+        group: ssl_certs
+        ansible_connection: local
+
+    - name: Install TXT records
+      ansible.builtin.include_role:
+        name: acme_install_txt_records
+
+    - name: Generate Certs
+      ansible.builtin.include_role:
+        name: acme_create_certs
+
+    - name: Remove TXT records
+      ansible.builtin.include_role:
+        name: acme_drop_txt_records

playbooks/acme-certs.yaml

@@ -0,0 +1,19 @@
+- hosts: ssl_certs:!disabled
+  become: true
+  roles:
+    - acme_request_certs
+
+- hosts: bridge.eco.tsi-dev.otc-service.com
+  become: true
+  roles:
+    - acme_install_txt_records
+
+- hosts: ssl_certs:!disabled
+  become: true
+  roles:
+    - acme_create_certs
+
+- hosts: bridge.eco.tsi-dev.otc-service.com
+  become: true
+  roles:
+    - acme_drop_txt_records

playbooks/apply-package-updates.yaml

@@ -0,0 +1,8 @@
+- hosts: "{{ target }}"
+  user: root
+  tasks:
+    - name: Run unattended-upgrade on debuntu
+      shell: |
+        unattended-upgrade -d
+      when: ansible_facts['os_family'] == "Debian"
+  # TODO add equivalent for other platforms

playbooks/base-ext.yaml

@@ -0,0 +1,5 @@
+- hosts: "!disabled"
+  name: "Configure base setup for all hosts - ext"
+  become: true
+  roles:
+    - base/audit

playbooks/base.yaml

@@ -0,0 +1,16 @@
+- hosts: "!disabled"
+  name: "Base: set up common environment on all hosts"
+  become: true
+  tasks:
+    - include_role: name=base/users
+    - block:
+        - include_role: name=base/repos
+        - include_role: name=base/server
+        - include_role: name=base/timezone
+        - include_role: name=base/unbound
+        - include_role: name=firewalld
+        - include_role: name=telegraf
+        - include_role: name=promtail
+      when:
+        # K8 cluster
+        - "ansible_connection != 'kubectl'"

playbooks/configure-vault-plugins.yaml

@@ -0,0 +1,12 @@
+- hosts: "vault1.eco.tsi-dev.otc-service.com"
+  become: true
+  name: "Vault: configure vault plugins"
+  environment:
+    VAULT_ADDR: "https://vault1.eco.tsi-dev.otc-service.com:8200"
+  tasks:
+    - include_role:
+        name: "hashivault"
+        tasks_from: "configure_plugins.yaml"
+      loop:  "{{ vault_plugins }}"
+      loop_control:
+        loop_var: "plugin"

playbooks/configure-vault.yaml

@@ -0,0 +1,10 @@
+---
+- hosts: vault-controller:!disabled
+  name: "Configure Vault instances"
+  tasks:
+    - name: Configure vault instance
+      include_role:
+        name: "configure_vault"
+      vars:
+        vault: "{{ item.value }}"
+      loop: "{{ vault_instances | dict2items }}"

playbooks/filter_plugins/getaddrinfo.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+
+
+class FilterModule(object):
+
+    def dns(self, value, family):
+        ret = set()
+        if family == '4':
+            match = 'has address'
+        elif family == '6':
+            match = 'has IPv6 address'
+        try:
+            # Note we use 'host' rather than something like
+            # getaddrinfo so we actually query DNS and don't get any
+            # local-only results from /etc/hosts
+            output = subprocess.check_output(
+                ['/usr/bin/host', value], universal_newlines=True)
+            for line in output.split('\n'):
+                if match in line:
+                    address = line.split()[-1]
+                    ret.add(address)
+        except Exception as e:
+            return ret
+        return sorted(ret)
+
+    def dns_a(self, value):
+        return self.dns(value, '4')
+
+    def dns_aaaa(self, value):
+        return self.dns(value, '6')
+
+    def filters(self):
+        return {
+            'dns_a': self.dns_a,
+            'dns_aaaa': self.dns_aaaa,
+        }

playbooks/install-helm-chart.yaml

@@ -0,0 +1,9 @@
+- hosts: "k8s-controller:!disabled"
+  name: "Install HELM chart"
+  become: true
+  tasks:
+    - include_role:
+        name: install-helm-chart
+      vars:
+        chart: "{{ helm_chart_instances[item] | combine((helm_chart_instances_secrets[item]|default({})), recursive=True) }}"
+      loop: "{{ helm_chart_instances | list }}"

playbooks/install-kubernetes.yaml

@@ -0,0 +1,4 @@
+---
+- hosts: all
+  roles:
+    - ensure-kubernetes

playbooks/library/cloud_user_group_assignment.py

@@ -0,0 +1,88 @@
+#!/usr/bin/python
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCUMENTATION = '''
+module: cloud_user_group_assignment
+extends_documentation_fragment: opentelekomcloud.cloud.otc
+description:
+  - Batch user group assignment
+options:
+  group:
+    description: Group name
+    type: str
+    required: true
+  users:
+    description: List of user names
+    type: list
+    elements: str
+  state:
+    description: Assignment state
+    type: str
+    choice: [present, absent]
+    default: present
+'''
+
+RETURN = '''
+'''
+
+EXAMPLES = '''
+'''
+
+import itertools
+
+from ansible_collections.opentelekomcloud.cloud.plugins.module_utils.otc import OTCModule
+
+
+class CloudUserGroupAssignmentModule(OTCModule):
+    argument_spec = dict(
+        group=dict(required=True, type='str'),
+        users=dict(required=True, type='list', elements='str'),
+        state=dict(type=str, choice=['present', 'absent'], default='present')
+    )
+
+    module_kwargs = dict(
+        supports_check_mode=True
+    )
+
+    def run(self):
+        group = self.conn.identity.find_group(name_or_id=self.params['group'])
+        changed = False
+
+        for user in self.params['users']:
+            usr = self.conn.identity.find_user(
+                name_or_id=user
+            )
+            is_in = self.conn.is_user_in_group(usr.id, group)
+            if self.params['state'] == 'present':
+                if not is_in:
+                    changed=True
+                    if not self.ansible.check_mode:
+                        self.conn.add_user_to_group(usr.id, group.id)
+            else:
+                if is_in:
+                    changed=True
+                    if not self.ansible.check_mode:
+                        self.conn.remove_user_from_group(usr.id, group.id)
+
+        self.exit_json(
+            changed=changed
+        )
+
+
+def main():
+    module = CloudUserGroupAssignmentModule()
+    module()
+
+
+if __name__ == '__main__':
+    main()

playbooks/library/scan_organizations.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python3
+
+import os
+
+import yaml
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            path=dict(type='str', required=True)
+        )
+    )
+
+    output = {}
+    for root, dirs, files in os.walk(module.params['path']):
+        for file in files:
+            current_root = os.path.basename(root)
+            a_yaml_file = open(os.path.join(root, file))
+            parsed_yaml_file = yaml.safe_load(a_yaml_file)
+            parent = os.path.basename(os.path.abspath(os.path.join(root, os.pardir)))
+            if parent in output:
+                if current_root in output[parent]:
+                    output[parent][current_root].update(parsed_yaml_file)
+                else:
+                    output[parent].update({current_root: parsed_yaml_file})
+            else:
+                output.update({parent: {current_root: parsed_yaml_file}})
+    module.exit_json(changed=True, data=output)
+
+
+if __name__ == '__main__':
+    main()

playbooks/roles/acme_create_certs/defaults/main.yaml

@@ -0,0 +1,4 @@
+certs_path: "/etc/ssl/{{ inventory_hostname }}"
+acme_directory: "https://acme-v02.api.letsencrypt.org/directory"
+acme_account_contact:
+  - "mailto:DL-PBCOTCDELECOCERT@t-systems.com"

playbooks/roles/acme_create_certs/handlers/reload-vault.yaml

@@ -0,0 +1,10 @@
+- name: Check vault process
+  command: pgrep -f vault
+  ignore_errors: yes
+  register: vault_pids
+
+- name: Reload Vault
+  ansible.builtin.service:
+    name: "vault"
+    state: "reloaded"
+  when: vault_pids.rc == 0

playbooks/roles/acme_create_certs/handlers/restart-gitea.yaml

@@ -0,0 +1,10 @@
+- name: Check gitea process
+  command: pgrep -f gitea
+  ignore_errors: yes
+  register: gitea_pids
+
+- name: Restart Gitea
+  ansible.builtin.service:
+    name: "gitea"
+    state: "restarted"
+  when: gitea_pids.rc == 0

playbooks/roles/acme_create_certs/handlers/restart-graphite.yaml

@@ -0,0 +1,10 @@
+- name: Check graphite process
+  command: pgrep -f graphite
+  ignore_errors: yes
+  register: graphite_pids
+
+- name: Restart Graphite
+  ansible.builtin.service:
+    name: "graphite"
+    state: "restarted"
+  when: graphite_pids.rc == 0

playbooks/roles/acme_create_certs/handlers/restart-haproxy.yaml

@@ -0,0 +1,10 @@
+- name: Check haproxy process
+  command: pgrep -f haproxy
+  ignore_errors: yes
+  register: haproxy_pids
+
+- name: Restart Haproxy
+  ansible.builtin.service:
+    name: "haproxy"
+    state: "restarted"
+  when: haproxy_pids.rc == 0

playbooks/roles/acme_create_certs/handlers/restart-keycloak.yaml

@@ -0,0 +1,10 @@
+- name: Check keycloak process
+  command: pgrep -f keycloak
+  ignore_errors: yes
+  register: keycloak_pids
+
+- name: Restart keycloak
+  ansible.builtin.service:
+    name: "keycloak"
+    state: "restarted"
+  when: keycloak_pids.rc == 0