diff --git a/cluster/ci/data/vars.yaml b/cluster/ci/data/vars.yaml index 9e4c0064b5cc3..84503b4d6b41b 100644 --- a/cluster/ci/data/vars.yaml +++ b/cluster/ci/data/vars.yaml @@ -12,9 +12,6 @@ openshift_additional_repos: [ # URLs and certs -provision_prefix: origin-ci- -gce_network_name: "origin-ci-ocp-network" - openshift_master_cluster_public_hostname: api.ci.openshift.org openshift_master_cluster_hostname: internal-master.ci.openshift.org public_hosted_zone: ci.openshift.org @@ -27,72 +24,32 @@ openshift_master_console_port: "8443" openshift_master_public_api_url: "https://api.ci.openshift.org" openshift_master_public_console_url: "https://api.ci.openshift.org/console" openshift_node_port_range: 30000-32000 +openshift_node_open_ports: [{"service":"Router stats port","port":"1936/tcp"},{"service":"Allowed open host ports","port":"9000-10000/tcp"},{"service":"Allowed open host ports","port":"9000-10000/udp"}] -openshift_master_named_certificates: [{"certfile": "{{ inventory_dir }}/ssl.crt", "keyfile": "{{ inventory_dir }}/ssl.key", "names": ["api.ci.openshift.org"]}] +openshift_master_named_certificates: [{"certfile": "{{ files_dir }}/ssl.crt", "keyfile": "{{ files_dir }}/ssl.key", "names": ["api.ci.openshift.org"]}] # Authentication and authorization -openshift_master_identity_providers: "{{ (lookup('file', '{{ inventory_dir }}/identity-providers.json' ) | default('{\"items\":[]}') | from_json).get('items') }}" +openshift_master_identity_providers: "{{ (lookup('file', '{{ files_dir }}/identity-providers.json' ) | default('{\"items\":[]}') | from_json).get('items') }}" provision_role_mappings: [{"user": "smarterclayton", "role": "cluster-admin"}] # Post config setting sizes openshift_hosted_router_replicas: 1 -openshift_hosted_router_certificate: {"certfile": "{{ inventory_dir }}/router.crt", "keyfile": "{{ inventory_dir }}/router.key", "cafile": "{{ inventory_dir }}/router.cacert"} +openshift_hosted_router_certificate: {"certfile": "{{ files_dir }}/router.crt", "keyfile": "{{ files_dir }}/router.key", "cafile": "{{ files_dir }}/router.cacert"} openshift_hosted_router_create_certificate: False openshift_hosted_registry_replicas: 1 openshift_schedulable: True # GCE provisioning info -# Project ID and zone settings for Google Cloud -gce_project_id: openshift-gce-devel -gce_region_name: us-central1 -gce_zone_name: us-central1-a -gce_service_account: ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com -gce_service_account_keyfile: "{{ inventory_dir }}/gce.json" -gce_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine - -openshift_hosted_registry_storage_gcs_keyfile: "{{ inventory_dir }}/gcs-registry.json" +openshift_hosted_registry_storage_gcs_keyfile: "{{ files_dir }}/gcs-registry.json" openshift_hosted_registry_storage_kind: object openshift_hosted_registry_storage_provider: gcs openshift_hosted_registry_storage_gcs_bucket: openshift-gce-devel-origin-ci-registry-bucket provision_gce_emptydir_quota: 10Gi - -# Control which node group router traffic is targeted at. -provision_gce_router_network_instance_group: ig-m # default: ig-i -provision_gce_node_groups: -- name: master - suffix: m - tags: ocp-master,preserve - machine_type: n1-standard-2 - boot_disk_size: 150 - scale: 1 -- name: node - suffix: n - tags: ocp-node,preserve - machine_type: n1-standard-8 - boot_disk_size: 150 - scale: 10 -- name: node-flex - suffix: nf - tags: ocp-node,preserve - machine_type: n1-standard-4 - boot_disk_size: 150 - scale: 0 - - -# An image or image family to pull from -provision_gce_registered_image: rhel-7 - provision_gce_docker_storage_driver: overlay2 -# Provide a startup script file to the GCE instances -provision_gce_startup_script_file: # "startup.sh" -# Provide userdata to the gce instances -provision_gce_user_data_file: - -rhsub_skip: true provision_custom_repositories: [ { "name": "oso-rhui-rhel-server-releases", @@ -126,10 +83,6 @@ provision_custom_repositories: [ } ] -############## -# New settings -############## - openshift_gcp_prefix: origin-ci- openshift_gcp_project: openshift-gce-devel @@ -139,7 +92,7 @@ openshift_gcp_zone: us-central1-a openshift_gcp_network_name: "{{ openshift_gcp_prefix }}network" openshift_gcp_iam_service_account: ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com -openshift_gcp_iam_service_account_keyfile: "{{ inventory_dir }}/gce.json" +openshift_gcp_iam_service_account_keyfile: "{{ files_dir }}/gce.json" openshift_gcp_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine @@ -148,7 +101,7 @@ openshift_gcp_infra_network_instance_group: ig-m openshift_gcp_image: 'rhel-7' openshift_gcp_base_image: 'centos-7' -openshift_gcp_registry_bucket_keyfile: "{{ inventory_dir }}/gcs-registry.json" +openshift_gcp_registry_bucket_keyfile: "{{ files_dir }}/gcs-registry.json" openshift_gcp_registry_bucket_name: openshift-gce-devel-origin-ci-registry-bucket openshift_gcp_node_group_config: diff --git a/cluster/test-deploy/data/.gitignore b/cluster/test-deploy/data/.gitignore index 4bfa3c666b81c..8192bcc3e02f8 100644 --- a/cluster/test-deploy/data/.gitignore +++ b/cluster/test-deploy/data/.gitignore @@ -1,3 +1,4 @@ * !.gitignore !vars.yml +!bootstrap-script.sh \ No newline at end of file diff --git a/cluster/test-deploy/data/bootstrap-script.sh b/cluster/test-deploy/data/bootstrap-script.sh new file mode 100644 index 0000000000000..a860fb7c0cc3e --- /dev/null +++ b/cluster/test-deploy/data/bootstrap-script.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +if [[ "$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap" -H "Metadata-Flavor: Google" )" != "true" ]]; then + echo "info: Bootstrap is not enabled for this instance, skipping" 1>&2 + exit 0 +fi + +if ! id=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-id" -H "Metadata-Flavor: Google" ); then + echo "error: Unable to get cluster-id for instance from cluster metadata" 1>&2 + exit 1 +fi + +if ! node_group=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/node-group" -H "Metadata-Flavor: Google" ); then + echo "error: Unable to get node-group for instance from cluster metadata" 1>&2 + exit 1 +fi + +if ! config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then + while true; do + if config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/project/attributes/${id}-bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then + break + fi + echo "info: waiting for ${id}-bootstrap-config to become available in cluster metadata ..." 1>&2 + sleep 5 + done +fi + +echo "Got bootstrap config from metadata" +mkdir -p /etc/origin/node +echo -n "${config}" > /etc/origin/node/bootstrap.kubeconfig +echo "BOOTSTRAP_CONFIG_NAME=node-config-${node_group}" >> /etc/sysconfig/origin-node +systemctl enable origin-node +systemctl start origin-node \ No newline at end of file diff --git a/cluster/test-deploy/data/vars.yaml b/cluster/test-deploy/data/vars.yaml index 826a3941e4b69..f9e3d7cdec578 100644 --- a/cluster/test-deploy/data/vars.yaml +++ b/cluster/test-deploy/data/vars.yaml @@ -12,9 +12,6 @@ openshift_additional_repos: [ # URLs and certs -provision_prefix: ci-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}- -gce_network_name: "ci-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}-ocp-network" - openshift_master_cluster_public_hostname: api.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.origin-ci-int-gce.dev.rhcloud.com openshift_master_cluster_hostname: internal-api.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.origin-ci-int-gce.dev.rhcloud.com wildcard_zone: apps.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.origin-ci-int-gce.dev.rhcloud.com @@ -35,7 +32,6 @@ openshift_node_open_ports: [{"service":"Router stats port","port":"1936/tcp"},{" # Authentication and authorization openshift_master_identity_providers: [{"name": "allow_all", "login": "true", "challenge": "true", "kind": "AllowAllPasswordIdentityProvider"}] -# provision_role_mappings: [{"user": "admin", "role": "cluster-admin"}] # Post config setting sizes @@ -43,54 +39,12 @@ openshift_hosted_router_replicas: 1 openshift_hosted_registry_replicas: 1 openshift_schedulable: True -# GCE provisioning info - -# Project ID and zone settings for Google Cloud -gce_project_id: openshift-gce-devel-ci -gce_region_name: us-east1 -gce_zone_name: us-east1-c -gce_service_account: jenkins-ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com -gce_service_account_keyfile: "{{ inventory_dir }}/gce.json" -gce_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine - -openshift_hosted_registry_storage_gcs_keyfile: "{{ inventory_dir }}/gce.json" +openshift_hosted_registry_storage_gcs_keyfile: "{{ files_dir }}/gce.json" openshift_hosted_registry_storage_kind: object openshift_hosted_registry_storage_provider: gcs openshift_hosted_registry_storage_gcs_bucket: origin-ci-test-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }} - -# Control which node group router traffic is targeted at. -provision_gce_router_network_instance_group: ig-m # default: ig-i -provision_gce_node_groups: -- name: master - suffix: m - tags: ocp-master - machine_type: n1-standard-2 - boot_disk_size: 150 - scale: 1 -- name: node - suffix: n - tags: ocp-node - machine_type: n1-standard-2 - boot_disk_size: 150 - scale: 3 -- name: node-flex - suffix: nf - tags: ocp-node - machine_type: n1-standard-2 - boot_disk_size: 150 - scale: 0 - -# An image or image family to pull from -provision_gce_registered_image: rhel-7 - provision_gce_docker_storage_driver: overlay2 -# Provide a startup script file to the GCE instances -provision_gce_startup_script_file: # "startup.sh" -# Provide userdata to the gce instances -provision_gce_user_data_file: - -rhsub_skip: true provision_custom_repositories: [ { "name": "oso-rhui-rhel-server-releases", @@ -124,10 +78,6 @@ provision_custom_repositories: [ } ] -############## -# New settings -############## - openshift_gcp_prefix: "ci-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}-" openshift_gcp_project: openshift-gce-devel-ci @@ -137,34 +87,57 @@ openshift_gcp_zone: us-east1-c openshift_gcp_network_name: "{{ openshift_gcp_prefix }}network" openshift_gcp_iam_service_account: jenkins-ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com -openshift_gcp_iam_service_account_keyfile: "{{ inventory_dir }}/gce.json" +openshift_gcp_iam_service_account_keyfile: "{{ files_dir }}/gce.json" openshift_gcp_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine openshift_gcp_infra_network_instance_group: ig-m -openshift_gcp_image: 'rhel-7' -openshift_gcp_base_image: 'centos-7' +# Instructs the launch job to build an AMI prior to cluster provisioning and then use that image +openshift_gcp_build_image: true +openshift_gcp_image: '{{ openshift_gcp_prefix }}images' +openshift_gcp_base_image: 'rhel-7' +openshift_gcp_root_image: 'centos-7' -openshift_gcp_registry_bucket_keyfile: "{{ inventory_dir }}/gce.json" +openshift_gcp_registry_bucket_keyfile: "{{ files_dir }}/gce.json" openshift_gcp_registry_bucket_name: "origin-ci-test-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}" +openshift_gcp_startup_script_file: "{{ files_dir }}/bootstrap-script.sh" openshift_gcp_node_group_config: - name: master suffix: m - tags: ocp-master + tags: ocp-master,ocp-infra-node,ocp-node machine_type: n1-standard-2 boot_disk_size: 150 scale: 1 + bootstrap: true + wait_for_stable: true - name: node suffix: n tags: ocp-node machine_type: n1-standard-2 boot_disk_size: 150 scale: 3 + bootstrap: true - name: node-flex suffix: nf tags: ocp-node machine_type: n1-standard-2 boot_disk_size: 150 scale: 0 + bootstrap: true + +openshift_master_node_configs: +- type: master + edits: + - key: kubeletArguments.node-labels + value: ['role=infra'] +- type: node + edits: + - key: kubeletArguments.node-labels + value: ['role=app'] + +osm_controller_args: {"experimental-cluster-signing-duration": ["20m"]} +openshift_master_bootstrap_enabled: true +openshift_master_bootstrap_auto_approve: true +openshift_master_bootstrap_auto_approver_node_selector: {"role": "infra"}