Do not remove files for bootstrap if resolv or dns.

Author: Kenny Woodson

playbooks/aws/README.md

@@ -43,10 +43,6 @@ The current expected work flow should be to provide an AMI with access to Opensh
 
 ```yaml
 ---
-# when creating an AMI set this to True
-# when installing a cluster set this to False
-openshift_node_bootstrap: True
-
 # specify a clusterid
 # openshift_aws_clusterid: default
 

playbooks/aws/openshift-cluster/build_ami.yml

@@ -51,6 +51,13 @@
     set_fact:
       ansible_ssh_user: root
 
+- hosts: nodes
+  gather_facts: no
+  tasks:
+  - name: set openshift_node_bootstrap to True for ami building
+    set_fact:
+      openshift_node_bootstrap: True
+
 - name: normalize groups
   include: ../../byo/openshift-cluster/initialize_groups.yml
 

playbooks/common/openshift-cluster/evaluate_groups.yml

@@ -51,7 +51,7 @@
     when:
     - g_etcd_hosts | default([]) | length not in [3,1]
     - not openshift_master_unsupported_embedded_etcd | default(False)
-    - not openshift_node_bootstrap | default(False)
+    - not (openshift_node_bootstrap | default(False))
 
   - name: Evaluate oo_all_hosts
     add_host:

roles/openshift_aws/defaults/main.yml

@@ -90,6 +90,10 @@ openshift_aws_node_group_config_node_volumes:
   delete_on_termination: True
 
 openshift_aws_node_group_config_tags: "{{ openshift_aws_clusterid | build_instance_tags(openshift_aws_kubernetes_cluster_status) }}"
+openshift_aws_node_group_termination_policy: Default
+openshift_aws_node_group_replace_instances: []
+openshift_aws_node_group_replace_all_instances: False
+openshift_aws_node_group_config_extra_labels: {}
 
 openshift_aws_node_group_config:
   tags: "{{ openshift_aws_node_group_config_tags }}"
@@ -106,7 +110,11 @@ openshift_aws_node_group_config:
     tags:
       host-type: master
       sub-host-type: default
+    labels:
+      type: master
     wait_for_instances: True
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
   compute:
     instance_type: m4.xlarge
     ami: "{{ openshift_aws_ami }}"
@@ -120,6 +128,10 @@ openshift_aws_node_group_config:
     tags:
       host-type: node
       sub-host-type: compute
+    labels:
+      type: compute
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
   infra:
     instance_type: m4.xlarge
     ami: "{{ openshift_aws_ami }}"
@@ -133,6 +145,10 @@ openshift_aws_node_group_config:
     tags:
       host-type: node
       sub-host-type: infra
+    labels:
+      type: infra
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
 
 openshift_aws_elb_security_groups:
 - "{{ openshift_aws_clusterid }}"
@@ -207,3 +223,27 @@ openshift_aws_vpc:
       az: "us-east-1e"
     - cidr: 172.31.16.0/20
       az: "us-east-1a"
+
+openshift_aws_node_config_location: /etc/origin/node/node-config.yaml
+openshift_aws_node_config_changes:
+- key: imageConfig.format
+  value: "{% raw %}registry.ops.openshift.com/openshift3/ose-${component}:${version}{% endraw %}"
+- key: kubeletArguments.cloud-config
+  value:
+  - /etc/origin/cloudprovider/aws.conf
+- key: kubeletArguments.cloud-provider
+  value:
+  - aws
+- key: kubeletArguments.kube-reserved
+  value:
+  - cpu=500m,memory=512M
+- key: kubeletArguments.system-reserved
+  value:
+  - cpu=500m,memory=512M
+- key: kubeletArguments.enable-controller-attach-detach
+  value:
+  - 'true'
+- key: networkConfig.mtu
+  value: 8951
+- key: networkConfig.networkPluginName
+  value: redhat/openshift-ovs-subnet

roles/openshift_aws/tasks/build_ami.yml

@@ -1,4 +1,8 @@
 ---
+- name: set openshift_node_bootstrap to True when building AMI
+  set_fact:
+    openshift_node_bootstrap: True
+
 - when: openshift_aws_create_vpc | bool
   name: create a vpc
   include: vpc.yml

roles/openshift_aws/tasks/launch_config.yml

@@ -22,23 +22,7 @@
     image_id: "{{ openshift_aws_ami }}"
     instance_type: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].instance_type }}"
     security_groups: "{{ ec2sgs.security_groups | map(attribute='group_id')| list }}"
-    user_data: |-
-      #cloud-config
-      {%  if openshift_aws_node_group_type != 'master' %}
-      write_files:
-      - path: /root/csr_kubeconfig
-        owner: root:root
-        permissions: '0640'
-        content: {{ openshift_aws_launch_config_bootstrap_token | default('') | to_yaml }}
-      - path: /root/openshift_settings
-        owner: root:root
-        permissions: '0640'
-        content:
-          openshift_type: "{{ openshift_aws_node_group_type }}"
-      runcmd:
-      - [ systemctl, enable, atomic-openshift-node]
-      - [ systemctl, start, atomic-openshift-node]
-      {% endif %}
+    user_data: "{{ lookup('template', 'user_data.j2') }}"
     key_name: "{{ openshift_aws_ssh_key_name }}"
     ebs_optimized: False
     volumes: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].volumes }}"

roles/openshift_aws/tasks/scale_group.yml

@@ -28,5 +28,7 @@
     load_balancers: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].elbs if 'elbs' in openshift_aws_node_group_config[openshift_aws_node_group_type] else omit }}"
     wait_for_instances: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].wait_for_instances | default(False)}}"
     vpc_zone_identifier: "{{ subnetout.subnets[0].id }}"
+    replace_instances: "{{ openshift_aws_node_group_replace_instances if openshift_aws_node_group_replace_instances != [] else omit }}"
+    replace_all_instances: "{{ omit if openshift_aws_node_group_replace_instances != [] else (openshift_aws_node_group_config[openshift_aws_node_group_type].replace_all_instances | default(omit)) }}"
     tags:
     - "{{ openshift_aws_node_group_config.tags | combine(openshift_aws_node_group_config[openshift_aws_node_group_type].tags) }}"

roles/openshift_aws/tasks/seal_ami.yml

@@ -1,4 +1,11 @@
 ---
+- name: Remove any ansible facts created during AMI creation
+  file:
+    path: "/etc/ansible/facts.d/{{ item }}"
+    state: absent
+  with_items:
+  - openshift.fact
+
 - name: fetch newly created instances
   ec2_remote_facts:
     region: "{{ openshift_aws_region }}"

roles/openshift_aws/templates/user_data.j2

@@ -0,0 +1,20 @@
+#cloud-config
+{%  if openshift_aws_node_group_type != 'master' %}
+write_files:
+- path: /root/csr_kubeconfig
+  owner: root:root
+  permissions: '0640'
+  content: {{ openshift_aws_launch_config_bootstrap_token | default('') | to_yaml }}
+- path: /root/openshift_settings.yaml
+  owner: root:root
+  permissions: '0640'
+  content:
+    openshift_group_type: {{ openshift_aws_node_group_type }}
+    openshift_group_type_labels: {{ openshift_aws_node_group_config_extra_labels | combine(openshift_aws_node_group_config[openshift_aws_node_group_type].labels)  }}
+    openshift_node_config_location: {{ openshift_aws_node_config_location }}
+    openshift_node_config_changes: {{ openshift_aws_node_config_changes | to_json }}
+runcmd:
+- [ ansible-playbook, /root/bootstrap.yml]
+- [ systemctl, enable, atomic-openshift-node]
+- [ systemctl, start, atomic-openshift-node]
+{% endif %}

roles/openshift_node/defaults/main.yml

@@ -11,6 +11,7 @@ openshift_node_ami_prep_packages:
 - "{{ openshift_service_type }}-node"
 - "{{ openshift_service_type }}-docker-excluder"
 - "{{ openshift_service_type }}-sdn-ovs"
+- openshift-ansible-roles
 - ansible
 - openvswitch
 - docker

roles/openshift_node/files/bootstrap.yml

@@ -0,0 +1,82 @@
+#!/usr/bin/ansible-playbook
+---
+- hosts: localhost
+  gather_facts: yes
+  vars:
+    origin_dns:
+      file: /etc/dnsmasq.d/origin-dns.conf
+      lines:
+      - regex: ^listen-address
+        state: absent
+      - regex: "^except-interface"
+        line: except-interface=lo
+        after: EOF
+        state: present
+    node_dns:
+      file: /etc/dnsmasq.d/node-dnsmasq.conf
+      lines:
+      - regex: "^server=/in-addr.arpa/127.0.0.1$"
+        line: server=/in-addr.arpa/127.0.0.1
+      - regex: "^server=/cluster.local/127.0.0.1$"
+        line: server=/cluster.local/127.0.0.1
+
+  tasks:
+  - include_vars: openshift_settings.yaml
+
+  - name: include yedit
+    include_role:
+      name: /usr/share/ansible/openshift-ansible/roles/lib_utils
+
+  - name: make yedit calls for node_config_changes
+    yedit:
+      src: "{{ openshift_node_config_location }}"
+      edits: "{{ openshift_node_config_changes }}"
+      backup: True
+
+  - name: fetch node labels
+    yedit:
+      src: "{{ openshift_node_config_location }}"
+      state: list
+      key: kubeletArguments.node-labels
+    register: yedout
+
+  - debug: var=yedout
+
+  - name: make yedit calls for node labels
+    yedit:
+      src: "{{ openshift_node_config_location }}"
+      key: kubeletArguments.node-labels
+      value: []
+      backup: True
+    when: yedout.result == None
+
+  - name: make yedit calls for node labels
+    yedit:
+      src: "{{ openshift_node_config_location }}"
+      key: kubeletArguments.node-labels
+      value: "{{ item.key | string }}={{ item.value | string }}"
+      update: "{{ True if yedout.result != None else omit }}"
+      append: "{{ True if yedout.result == None else omit }}"
+      curr_value: "{{ item.key | string }}={{ item.value | string }}"
+      backup: True
+    with_dict: "{{ openshift_group_type_labels }}"
+
+  - name: set the data for node_dns
+    lineinfile:
+      create: yes
+      insertafter: EOF
+      path: "{{ node_dns.file }}"
+      regexp: "{{ item.regex }}"
+      line: "{{ item.line | default(omit) }}"
+    with_items: "{{ node_dns.lines }}"
+
+  - name: set the data for origin_dns
+    lineinfile:
+      create: yes
+      state: "{{ item.state | default('present') }}"
+      insertafter: "{{ item.after | default(omit) }}"
+      path: "{{ origin_dns.file }}"
+      regexp: "{{ item.regex }}"
+      line: "{{ item.line | default(omit)}}"
+    with_items: "{{ origin_dns.lines }}"
+

roles/openshift_node/tasks/bootstrap.yml

@@ -42,6 +42,11 @@
     path: /etc/origin/.config_managed
   register: rpmgenerated_config
 
+- name: laydown the bootstrap.yml file for on boot configuration
+  copy:
+    src: bootstrap.yml
+    dest: /root/bootstrap.yml
+
 - when: rpmgenerated_config.stat.exists
   block:
   - name: Remove RPM generated config files if present
@@ -50,6 +55,7 @@
       state: absent
     with_items:
     - master
+    - .config_managed
 
   # with_fileglob doesn't work correctly due to a few issues.
   # Could change this to fileglob when it gets fixed.
@@ -62,5 +68,7 @@
     file:
       path: "{{ item.path }}"
       state: absent
-    when: "'resolv.conf' not in item.path or 'node-dnsmasq.conf' not in item.path"
+    when:
+    - "'resolv.conf' not in item.path"
+    - "'node-dnsmasq.conf' not in item.path"
     with_items: "{{ find_results.files }}"