Merge pull request #5361 from kwoodson/fix_bootstrap_files

Provisioning updates.

Author: sdodson

playbooks/aws/openshift-cluster/accept.yml

@@ -42,12 +42,12 @@
     until: "'instances' in instancesout and instancesout.instances|length > 0"
 
   - debug:
-      msg: "{{ instancesout.instances|map(attribute='private_dns_name') | list | regex_replace('.ec2.internal') }}"
+      msg: "{{ instancesout.instances|map(attribute='private_dns_name') | list }}"
 
   - name: approve nodes
     oc_adm_csr:
       #approve_all: True
-      nodes: "{{ instancesout.instances|map(attribute='private_dns_name') | list | regex_replace('.ec2.internal') }}"
-      timeout: 0
+      nodes: "{{ instancesout.instances|map(attribute='private_dns_name') | list  }}"
+      timeout: 60
     register: nodeout
     delegate_to: "{{ mastersout.instances[0].public_ip_address }}"

playbooks/aws/openshift-cluster/build_ami.yml

@@ -27,6 +27,7 @@
   - name: set the user to perform installation
     set_fact:
       ansible_ssh_user: "{{ openshift_aws_build_ami_ssh_user | default('root') }}"
+      openshift_node_bootstrap: True
 
 # This is the part that installs all of the software and configs for the instance
 # to become a node.

playbooks/common/openshift-cluster/evaluate_groups.yml

@@ -51,7 +51,7 @@
     when:
     - g_etcd_hosts | default([]) | length not in [3,1]
     - not openshift_master_unsupported_embedded_etcd | default(False)
-    - not openshift_node_bootstrap | default(False)
+    - not (openshift_node_bootstrap | default(False))
 
   - name: Evaluate oo_all_hosts
     add_host:

playbooks/common/openshift-node/image_prep.yml

@@ -2,13 +2,13 @@
 - name: normalize groups
   include: ../../byo/openshift-cluster/initialize_groups.yml
 
-- name: run the std_include
+- name: evaluate the groups
   include: ../openshift-cluster/evaluate_groups.yml
 
-- name: run the std_include
+- name: initialize the facts
   include: ../openshift-cluster/initialize_facts.yml
 
-- name: run the std_include
+- name: initialize the repositories
   include: ../openshift-cluster/initialize_openshift_repos.yml
 
 - name: run node config setup

roles/lib_openshift/library/oc_adm_csr.py

@@ -1478,11 +1478,23 @@ def action_needed(csr, action):
 
         return False
 
+    def get_csr_request(self, request):
+        '''base64 decode the request object and call openssl to determine the
+           subject and specifically the CN: from the request
+
+           Output:
+           (0, '...
+                Subject: O=system:nodes, CN=system:node:ip-172-31-54-54.ec2.internal
+                ...')
+        '''
+        import base64
+        return self._run(['openssl', 'req', '-noout', '-text'], base64.b64decode(request))[1]
+
     def match_node(self, csr):
         '''match an inc csr to a node in self.nodes'''
         for node in self.nodes:
-            # we have a match
-            if node['name'] in csr['metadata']['name']:
+            # we need to match based upon the csr's request certificate's CN
+            if node['name'] in self.get_csr_request(csr['spec']['request']):
                 node['csrs'][csr['metadata']['name']] = csr
 
                 # check that the username is the node and type is 'Approved'

roles/lib_openshift/src/class/oc_adm_csr.py

@@ -66,11 +66,23 @@ def action_needed(csr, action):
 
         return False
 
+    def get_csr_request(self, request):
+        '''base64 decode the request object and call openssl to determine the
+           subject and specifically the CN: from the request
+
+           Output:
+           (0, '...
+                Subject: O=system:nodes, CN=system:node:ip-172-31-54-54.ec2.internal
+                ...')
+        '''
+        import base64
+        return self._run(['openssl', 'req', '-noout', '-text'], base64.b64decode(request))[1]
+
     def match_node(self, csr):
         '''match an inc csr to a node in self.nodes'''
         for node in self.nodes:
-            # we have a match
-            if node['name'] in csr['metadata']['name']:
+            # we need to match based upon the csr's request certificate's CN
+            if node['name'] in self.get_csr_request(csr['spec']['request']):
                 node['csrs'][csr['metadata']['name']] = csr
 
                 # check that the username is the node and type is 'Approved'

roles/openshift_aws/defaults/main.yml

@@ -4,7 +4,6 @@ openshift_aws_create_iam_cert: True
 openshift_aws_create_security_groups: True
 openshift_aws_create_launch_config: True
 openshift_aws_create_scale_group: True
-openshift_aws_kubernetes_cluster_status: owned  # or shared
 openshift_aws_node_group_type: master
 
 openshift_aws_wait_for_ssh: True
@@ -13,6 +12,7 @@ openshift_aws_clusterid: default
 openshift_aws_region: us-east-1
 openshift_aws_vpc_name: "{{ openshift_aws_clusterid }}"
 openshift_aws_build_ami_group: "{{ openshift_aws_clusterid }}"
+openshift_aws_kubernetes_cluster_status: "{{ openshift_aws_clusterid }}"
 
 openshift_aws_iam_cert_name: "{{ openshift_aws_clusterid }}-master-external"
 openshift_aws_iam_cert_path: ''
@@ -89,6 +89,10 @@ openshift_aws_node_group_config_node_volumes:
   delete_on_termination: True
 
 openshift_aws_node_group_config_tags: "{{ openshift_aws_clusterid | build_instance_tags(openshift_aws_kubernetes_cluster_status) }}"
+openshift_aws_node_group_termination_policy: Default
+openshift_aws_node_group_replace_instances: []
+openshift_aws_node_group_replace_all_instances: False
+openshift_aws_node_group_config_extra_labels: {}
 
 openshift_aws_node_group_config:
   tags: "{{ openshift_aws_node_group_config_tags }}"
@@ -105,7 +109,11 @@ openshift_aws_node_group_config:
     tags:
       host-type: master
       sub-host-type: default
+    labels:
+      type: master
     wait_for_instances: True
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
   compute:
     instance_type: m4.xlarge
     ami: "{{ openshift_aws_ami }}"
@@ -119,6 +127,10 @@ openshift_aws_node_group_config:
     tags:
       host-type: node
       sub-host-type: compute
+    labels:
+      type: compute
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
   infra:
     instance_type: m4.xlarge
     ami: "{{ openshift_aws_ami }}"
@@ -132,6 +144,10 @@ openshift_aws_node_group_config:
     tags:
       host-type: node
       sub-host-type: infra
+    labels:
+      type: infra
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
 
 openshift_aws_elb_security_groups:
 - "{{ openshift_aws_clusterid }}"
@@ -211,3 +227,7 @@ openshift_aws_vpc:
       az: "us-east-1e"
     - cidr: 172.31.16.0/20
       az: "us-east-1a"
+
+openshift_aws_node_run_bootstrap_startup: True
+openshift_aws_node_user_data: ''
+openshift_aws_node_config_namespace: openshift-node

roles/openshift_aws/tasks/launch_config.yml

@@ -4,6 +4,11 @@
   when:
   - openshift_aws_ami is undefined
 
+- fail:
+    msg: "Ensure that openshift_deployment_type is defined."
+  when:
+  - openshift_deployment_type is undefined
+
 - name: query vpc
   ec2_vpc_net_facts:
     region: "{{ openshift_aws_region }}"
@@ -27,23 +32,7 @@
     image_id: "{{ openshift_aws_ami }}"
     instance_type: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].instance_type }}"
     security_groups: "{{ openshift_aws_launch_config_security_group_id  | default(ec2sgs.security_groups | map(attribute='group_id')| list) }}"
-    user_data: |-
-      #cloud-config
-      {%  if openshift_aws_node_group_type != 'master' %}
-      write_files:
-      - path: /root/csr_kubeconfig
-        owner: root:root
-        permissions: '0640'
-        content: {{ openshift_aws_launch_config_bootstrap_token | default('') | to_yaml }}
-      - path: /root/openshift_settings
-        owner: root:root
-        permissions: '0640'
-        content:
-          openshift_type: "{{ openshift_aws_node_group_type }}"
-      runcmd:
-      - [ systemctl, enable, atomic-openshift-node]
-      - [ systemctl, start, atomic-openshift-node]
-      {% endif %}
+    user_data: "{{ lookup('template', 'user_data.j2') }}"
     key_name: "{{ openshift_aws_ssh_key_name }}"
     ebs_optimized: False
     volumes: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].volumes }}"

roles/openshift_aws/tasks/provision_instance.yml

@@ -1,4 +1,8 @@
 ---
+- name: set openshift_node_bootstrap to True when building AMI
+  set_fact:
+    openshift_node_bootstrap: True
+
 - name: query vpc
   ec2_vpc_net_facts:
     region: "{{ openshift_aws_region }}"
@@ -53,10 +57,6 @@
     timeout: 300
     search_regex: OpenSSH
 
-- name: Pause 10 seconds to ensure ssh actually accepts logins
-  pause:
-    seconds: 20
-
 - name: add host to nodes
   add_host:
     groups: nodes

roles/openshift_aws/tasks/scale_group.yml

@@ -28,5 +28,7 @@
     load_balancers: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].elbs if 'elbs' in openshift_aws_node_group_config[openshift_aws_node_group_type] else omit }}"
     wait_for_instances: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].wait_for_instances | default(False)}}"
     vpc_zone_identifier: "{{ subnetout.subnets[0].id }}"
+    replace_instances: "{{ openshift_aws_node_group_replace_instances if openshift_aws_node_group_replace_instances != [] else omit }}"
+    replace_all_instances: "{{ omit if openshift_aws_node_group_replace_instances != [] else (openshift_aws_node_group_config[openshift_aws_node_group_type].replace_all_instances | default(omit)) }}"
     tags:
     - "{{ openshift_aws_node_group_config.tags | combine(openshift_aws_node_group_config[openshift_aws_node_group_type].tags) }}"

roles/openshift_aws/tasks/seal_ami.yml

@@ -1,4 +1,11 @@
 ---
+- name: Remove any ansible facts created during AMI creation
+  file:
+    path: "/etc/ansible/facts.d/{{ item }}"
+    state: absent
+  with_items:
+  - openshift.fact
+
 - name: fetch newly created instances
   ec2_remote_facts:
     region: "{{ openshift_aws_region }}"

roles/openshift_aws/templates/user_data.j2

@@ -0,0 +1,26 @@
+{% if openshift_aws_node_user_data is defined and openshift_aws_node_user_data != '' %}
+{{ openshift_aws_node_user_data }}
+{% else %}
+#cloud-config
+write_files:
+- path: /root/openshift_bootstrap/openshift_settings.yaml
+  owner: 'root:root'
+  permissions: '0640'
+  content: |
+    openshift_group_type: {{ openshift_aws_node_group_type }}
+{%   if openshift_aws_node_group_type != 'master' %}
+- path: /etc/origin/node/csr_kubeconfig
+  owner: 'root:root'
+  permissions: '0640'
+  encoding: b64
+  content: {{ openshift_aws_launch_config_bootstrap_token | b64encode }}
+{%   endif %}
+runcmd:
+{%     if openshift_aws_node_run_bootstrap_startup %}
+- [ ansible-playbook, /root/openshift_bootstrap/bootstrap.yml]
+{%     endif %}
+{%     if openshift_aws_node_group_type != 'master' %}
+- [ systemctl, enable, {% if openshift_deployment_type == 'openshift-enterprise' %}atomic-openshift{% else %}origin{% endif %}-node]
+- [ systemctl, start, {% if openshift_deployment_type == 'openshift-enterprise' %}atomic-openshift{% else %}origin{% endif %}-node]
+{%     endif %}
+{% endif %}

roles/openshift_master/defaults/main.yml

@@ -54,3 +54,88 @@ r_openshift_master_sdn_network_plugin_name: "{{ r_openshift_master_sdn_network_p
 
 openshift_master_image_config_latest_default: "{{ openshift_image_config_latest | default(False) }}"
 openshift_master_image_config_latest: "{{ openshift_master_image_config_latest_default }}"
+
+openshift_master_config_dir_default: "{{ (openshift.common.config_base | default('/etc/origin/master')) ~ '/master' }}"
+openshift_master_config_dir: "{{ openshift_master_config_dir_default }}"
+openshift_master_cloud_provider: "{{ openshift_cloudprovider_kind | default('aws') }}"
+
+openshift_master_node_config_networkconfig_mtu: 1450
+
+openshift_master_node_config_kubeletargs_cpu: 500m
+openshift_master_node_config_kubeletargs_mem: 512M
+
+openshift_master_bootstrap_enabled: False
+
+openshift_master_client_binary: "{{ openshift.common.client_binary if openshift is defined else 'oc' }}"
+
+openshift_master_config_imageconfig_format: "{{ oreg_url if oreg_url != '' else 'registry.access.redhat.com/openshift3/ose-${component}:${version}' }}"
+
+# these are for the default settings in a generated node-config.yaml
+openshift_master_node_config_default_edits:
+- key: nodeName
+  state: absent
+- key: dnsBindAddress
+  value: 127.0.0.1:53
+- key: dnsDomain
+  value: cluster.local
+- key: dnsRecursiveResolvConf
+  value: /etc/origin/node/resolv.conf
+- key: imageConfig.format
+  value: "{{ openshift_master_config_imageconfig_format }}"
+- key: kubeletArguments.cloud-config
+  value:
+  - "/etc/origin/cloudprovider/{{ openshift_master_cloud_provider }}.conf"
+- key: kubeletArguments.cloud-provider
+  value:
+  - "{{ openshift_master_cloud_provider }}"
+- key: kubeletArguments.kube-reserved
+  value:
+  - "cpu={{ openshift_master_node_config_kubeletargs_cpu }},memory={{ openshift_master_node_config_kubeletargs_mem }}"
+- key: kubeletArguments.system-reserved
+  value:
+  - "cpu={{ openshift_master_node_config_kubeletargs_cpu }},memory={{ openshift_master_node_config_kubeletargs_mem }}"
+- key: enable-controller-attach-detach
+  value:
+  - 'true'
+- key: networkConfig.mtu
+  value: 8951
+- key: networkConfig.networkPluginName
+  value: "{{ r_openshift_master_sdn_network_plugin_name }}"
+- key: networkPluginName
+  value: "{{ r_openshift_master_sdn_network_plugin_name }}"
+
+
+# We support labels for all nodes here
+openshift_master_node_config_kubeletargs_default_labels: []
+# We do support overrides for node group labels
+openshift_master_node_config_kubeletargs_master_labels: []
+openshift_master_node_config_kubeletargs_infra_labels: []
+openshift_master_node_config_kubeletargs_compute_labels: []
+
+openshift_master_node_config_master:
+  type: master
+  edits:
+  - key: kubeletArguments.node-labels
+    value: "{{ openshift_master_node_config_kubeletargs_default_labels |
+               union(openshift_master_node_config_kubeletargs_master_labels) |
+               union(['type=master']) }}"
+openshift_master_node_config_infra:
+  type: infra
+  edits:
+  - key: kubeletArguments.node-labels
+    value: "{{ openshift_master_node_config_kubeletargs_default_labels |
+               union(openshift_master_node_config_kubeletargs_infra_labels) |
+               union(['type=infra']) }}"
+openshift_master_node_config_compute:
+  type: compute
+  edits:
+  - key: kubeletArguments.node-labels
+    value: "{{ openshift_master_node_config_kubeletargs_default_labels |
+               union(openshift_master_node_config_kubeletargs_compute_labels) |
+               union(['type=compute']) }}"
+
+openshift_master_node_configs:
+- "{{ openshift_master_node_config_infra }}"
+- "{{ openshift_master_node_config_compute }}"
+
+openshift_master_bootstrap_namespace: openshift-node

roles/openshift_master/meta/main.yml

@@ -13,4 +13,5 @@ galaxy_info:
   - cloud
 dependencies:
 - role: lib_openshift
+- role: lib_utils
 - role: lib_os_firewall