Skip to content

Commit 5285f89

Browse files
yuwang-RHyuwang-RH
committed
OCM-14545 | ci: Support security group and proxy settings for hosted-cp shared vpc profile
1 parent 4d281ae commit 5285f89

File tree

6 files changed

+251
-38
lines changed

6 files changed

+251
-38
lines changed

‎tests/ci/data/profiles/rosa-hcp.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ profiles:
176176
autoscale: true
177177
kms_key: true
178178
networking: true
179-
proxy_enabled: false
179+
proxy_enabled: true
180180
label_enabled: false
181181
zones: ""
182182
tag_enabled: true
@@ -188,7 +188,7 @@ profiles:
188188
volume_size: 75
189189
disable_uwm: true
190190
autoscaler_enabled: false
191-
additional_sg_number: 0
191+
additional_sg_number: 3
192192
registries_config: true
193193
blocked_registries: true
194194
account-role:

‎tests/utils/handler/cluster_handler.go

Lines changed: 50 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -538,8 +538,11 @@ func (ch *clusterHandler) GenerateClusterCreateFlags() ([]string, error) {
538538
ch.clusterConfig.Networking = networking
539539
}
540540
if ch.profile.ClusterConfig.BYOVPC {
541-
var vpc *vpc_client.VPC
542-
var err error
541+
var (
542+
vpc *vpc_client.VPC
543+
err error
544+
securityGroups []string
545+
)
543546
vpcPrefix := helper.TrimNameByLength(clusterName, 20)
544547
log.Logger.Info("Got BYOVPC set to true. Going to prepare subnets")
545548
cidrValue := constants.DefaultVPCCIDRValue
@@ -572,15 +575,57 @@ func (ch *clusterHandler) GenerateClusterCreateFlags() ([]string, error) {
572575
}
573576
flags = append(flags,
574577
"--subnet-ids", subnetsFlagValue)
578+
if ch.profile.ClusterConfig.AdditionalSGNumber != 0 {
579+
securityGroups, err = resourcesHandler.
580+
PrepareAdditionalSecurityGroups(ch.profile.ClusterConfig.AdditionalSGNumber, vpcPrefix)
581+
if err != nil {
582+
return flags, err
583+
}
584+
computeSGs := strings.Join(securityGroups, ",")
585+
infraSGs := strings.Join(securityGroups, ",")
586+
controlPlaneSGs := strings.Join(securityGroups, ",")
587+
if ch.profile.ClusterConfig.HCP {
588+
flags = append(flags,
589+
"--additional-compute-security-group-ids", computeSGs,
590+
)
591+
ch.clusterConfig.AdditionalSecurityGroups = &ClusterConfigure.AdditionalSecurityGroups{
592+
WorkerSecurityGroups: computeSGs,
593+
}
594+
} else {
595+
flags = append(flags,
596+
"--additional-infra-security-group-ids", infraSGs,
597+
"--additional-control-plane-security-group-ids", controlPlaneSGs,
598+
"--additional-compute-security-group-ids", computeSGs,
599+
)
600+
ch.clusterConfig.AdditionalSecurityGroups = &ClusterConfigure.AdditionalSecurityGroups{
601+
ControlPlaneSecurityGroups: controlPlaneSGs,
602+
InfraSecurityGroups: infraSGs,
603+
WorkerSecurityGroups: computeSGs,
604+
}
605+
}
606+
}
575607

576608
if ch.profile.ClusterConfig.SharedVPC {
577-
subnetArns, err := resourcesHandler.PrepareSubnetArns(subnetsFlagValue)
609+
var (
610+
securityGroupARns []string
611+
sharedResourceArns []string
612+
)
613+
sharedResourceArns, err = resourcesHandler.PrepareSubnetArns(subnetsFlagValue)
578614
if err != nil {
579615
return flags, err
580616
}
581617

618+
if len(securityGroups) > 0 {
619+
securityGroupARns, err = resourcesHandler.PrepareSecurityGroupArns(securityGroups, true)
620+
621+
if err != nil {
622+
return flags, err
623+
}
624+
sharedResourceArns = append(sharedResourceArns, securityGroupARns...)
625+
}
626+
582627
resourceShareName := fmt.Sprintf("%s-%s", sharedVPCRolePrefix, "resource-share")
583-
_, err = resourcesHandler.PrepareResourceShare(resourceShareName, subnetArns)
628+
_, err = resourcesHandler.PrepareResourceShare(resourceShareName, sharedResourceArns)
584629
if err != nil {
585630
return flags, err
586631
}
@@ -626,35 +671,7 @@ func (ch *clusterHandler) GenerateClusterCreateFlags() ([]string, error) {
626671
}
627672
}
628673
}
629-
if ch.profile.ClusterConfig.AdditionalSGNumber != 0 {
630-
securityGroups, err := resourcesHandler.
631-
PrepareAdditionalSecurityGroups(ch.profile.ClusterConfig.AdditionalSGNumber, vpcPrefix)
632-
if err != nil {
633-
return flags, err
634-
}
635-
computeSGs := strings.Join(securityGroups, ",")
636-
infraSGs := strings.Join(securityGroups, ",")
637-
controlPlaneSGs := strings.Join(securityGroups, ",")
638-
if ch.profile.ClusterConfig.HCP {
639-
flags = append(flags,
640-
"--additional-compute-security-group-ids", computeSGs,
641-
)
642-
ch.clusterConfig.AdditionalSecurityGroups = &ClusterConfigure.AdditionalSecurityGroups{
643-
WorkerSecurityGroups: computeSGs,
644-
}
645-
} else {
646-
flags = append(flags,
647-
"--additional-infra-security-group-ids", infraSGs,
648-
"--additional-control-plane-security-group-ids", controlPlaneSGs,
649-
"--additional-compute-security-group-ids", computeSGs,
650-
)
651-
ch.clusterConfig.AdditionalSecurityGroups = &ClusterConfigure.AdditionalSecurityGroups{
652-
ControlPlaneSecurityGroups: controlPlaneSGs,
653-
InfraSecurityGroups: infraSGs,
654-
WorkerSecurityGroups: computeSGs,
655-
}
656-
}
657-
}
674+
658675
if ch.profile.ClusterConfig.ProxyEnabled {
659676
proxyName := vpc.VPCName
660677
if proxyName == "" {
@@ -679,7 +696,6 @@ func (ch *clusterHandler) GenerateClusterCreateFlags() ([]string, error) {
679696
"--no-proxy", proxy.NoProxy,
680697
"--additional-trust-bundle-file", proxy.CABundleFilePath,
681698
)
682-
683699
}
684700
}
685701
if ch.profile.ClusterConfig.BillingAccount != "" {

‎tests/utils/handler/interface.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ type Resources struct {
8484
VpcID string `json:"vpc_id,omitempty"`
8585
HCPRoute53ShareRole string `json:"hcp_route53_share_role,omitempty"`
8686
HCPVPCEndpointShareRole string `json:"hcp_vpc_endpoint_share_role,omitempty"`
87+
ProxyInstanceID string `json:"proxy_instance_id,omitempty"`
8788
}
8889

8990
type FromSharedAWSAccount struct {
@@ -108,4 +109,5 @@ type ProxyDetail struct {
108109
HTTPProxy string
109110
CABundleFilePath string
110111
NoProxy string
112+
InstanceID string
111113
}

‎tests/utils/handler/resources_handler.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,17 @@ func (rh *resourcesHandler) DestroyResources() (errors []error) {
208208
rh.registerDNSDomain("")
209209
}
210210
}
211+
// Delete proxy resourses
212+
if resources.ProxyInstanceID != "" {
213+
err = rh.CleanupProxyResources(
214+
resources.ProxyInstanceID,
215+
resources.FromSharedAWSAccount != nil && resources.FromSharedAWSAccount.VPC,
216+
)
217+
success := destroyLog(err, "proxy resources")
218+
if success {
219+
rh.registerProxyInstanceID("")
220+
}
221+
}
211222
// delete resource share
212223
if resources.ResourceShareArn != "" {
213224
log.Logger.Infof("Find prepared resource share: %s", resources.ResourceShareArn)
@@ -453,6 +464,11 @@ func (rh *resourcesHandler) registerVPC(vpc *vpc_client.VPC) error {
453464
return rh.saveToFile()
454465
}
455466

467+
func (rh *resourcesHandler) registerProxyInstanceID(proxyInsID string) error {
468+
rh.resources.ProxyInstanceID = proxyInsID
469+
return rh.saveToFile()
470+
}
471+
456472
func (rh *resourcesHandler) GetVPC() *vpc_client.VPC {
457473
return rh.vpc
458474
}

‎tests/utils/handler/resources_handler_clean.go

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,15 @@
11
package handler
22

33
import (
4+
"context"
5+
"fmt"
46
"strings"
7+
"time"
58

9+
"k8s.io/apimachinery/pkg/util/wait"
10+
11+
"github.com/aws/aws-sdk-go-v2/service/ec2"
12+
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
613
"github.com/openshift-online/ocm-common/pkg/aws/aws_client"
714
"github.com/openshift-online/ocm-common/pkg/test/kms_key"
815
"github.com/openshift-online/ocm-common/pkg/test/vpc_client"
@@ -151,3 +158,146 @@ func (rh *resourcesHandler) DeleteAccountRoles() error {
151158
}
152159
return nil
153160
}
161+
162+
func (rh *resourcesHandler) GetEIPAssociationAndAllocationIDsByInstanceID(
163+
publicIP string, sharedVPC bool,
164+
) (string, string, error) {
165+
var (
166+
err error
167+
awsClient *aws_client.AWSClient
168+
)
169+
if sharedVPC {
170+
awsClient, err = rh.GetAWSClient(true)
171+
} else {
172+
awsClient, err = rh.GetAWSClient(false)
173+
}
174+
if err != nil {
175+
log.Logger.Errorf("Get AWS Client failed: %s", err)
176+
return "", "", err
177+
}
178+
input := &ec2.DescribeAddressesInput{
179+
PublicIps: []string{publicIP},
180+
}
181+
output, err := awsClient.Ec2Client.DescribeAddresses(context.TODO(), input)
182+
if err != nil {
183+
log.Logger.Errorf("Failed to describe addresses: %s", err)
184+
return "", "", err
185+
}
186+
187+
if len(output.Addresses) == 0 {
188+
return "", "", fmt.Errorf("no EIP association found for instance ID %s", publicIP)
189+
}
190+
191+
associationID := *output.Addresses[0].AssociationId
192+
allocationID := *output.Addresses[0].AllocationId
193+
return associationID, allocationID, nil
194+
}
195+
196+
func (rh *resourcesHandler) CleanupProxyResources(instID string, sharedVPC bool) error {
197+
var (
198+
err error
199+
awsClient *aws_client.AWSClient
200+
)
201+
202+
if sharedVPC {
203+
awsClient, err = rh.GetAWSClient(true)
204+
} else {
205+
awsClient, err = rh.GetAWSClient(false)
206+
}
207+
if err != nil {
208+
log.Logger.Errorf("Get AWS Client failed: %s", err)
209+
return err
210+
}
211+
insOut, err := awsClient.Ec2Client.DescribeInstances(
212+
context.TODO(),
213+
&ec2.DescribeInstancesInput{
214+
InstanceIds: []string{instID},
215+
},
216+
)
217+
if err != nil {
218+
log.Logger.Errorf("Describe proxy instance failed: %s", err)
219+
return err
220+
}
221+
if len(insOut.Reservations) == 0 || len(insOut.Reservations[0].Instances) == 0 {
222+
err = fmt.Errorf("instance %s not found", rh.resources.ProxyInstanceID)
223+
return err
224+
}
225+
instance := insOut.Reservations[0].Instances[0]
226+
// release EIP
227+
associationID, allocationID, err := rh.GetEIPAssociationAndAllocationIDsByInstanceID(
228+
*instance.PublicIpAddress, sharedVPC,
229+
)
230+
keyName := *instance.KeyName
231+
SGID := *instance.SecurityGroups[0].GroupId
232+
if err != nil {
233+
log.Logger.Errorf("Get EIP Association ID failed: %s", err)
234+
return err
235+
}
236+
_, err = awsClient.DisassociateAddress(associationID)
237+
if err != nil {
238+
log.Logger.Errorf("Disassociate Addrress failed: %s", err)
239+
return err
240+
}
241+
_, err = awsClient.ReleaseAddress(allocationID)
242+
if err != nil {
243+
log.Logger.Errorf("Release Address failed: %s", err)
244+
return err
245+
}
246+
log.Logger.Infof("Released EIP: %s", allocationID)
247+
248+
// terminate proxy instance
249+
_, err = awsClient.Ec2Client.TerminateInstances(context.TODO(), &ec2.TerminateInstancesInput{
250+
InstanceIds: []string{instID},
251+
})
252+
if err != nil {
253+
log.Logger.Errorf("Terminate instance failed: %s", err)
254+
return err
255+
}
256+
log.Logger.Infof("Terminating instance: %s", instID)
257+
258+
err = wait.PollUntilContextTimeout(
259+
context.TODO(),
260+
30*time.Second,
261+
10*time.Minute,
262+
false,
263+
func(ctx context.Context) (bool, error) {
264+
output, err := awsClient.Ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{
265+
InstanceIds: []string{instID},
266+
})
267+
if err != nil {
268+
return false, err
269+
}
270+
if len(output.Reservations) == 0 || len(output.Reservations[0].Instances) == 0 {
271+
return false, nil
272+
}
273+
instance := output.Reservations[0].Instances[0]
274+
if instance.State != nil && instance.State.Name == types.InstanceStateNameTerminated {
275+
return true, nil
276+
}
277+
return false, nil
278+
},
279+
)
280+
if err != nil {
281+
log.Logger.Errorf("Instance termination confirmation failed: %s", err)
282+
return err
283+
}
284+
log.Logger.Infof("Instance %s is terminated", instID)
285+
286+
// Delete secrity group
287+
_, err = awsClient.DeleteSecurityGroup(SGID)
288+
if err != nil {
289+
log.Logger.Errorf("Delete security group failed: %s", err)
290+
return err
291+
}
292+
log.Logger.Infof("Deleted security group: %s", SGID)
293+
294+
// Delete key pair
295+
_, err = awsClient.DeleteKeyPair(keyName)
296+
if err != nil {
297+
log.Logger.Errorf("Delete key pair failed: %s", err)
298+
return err
299+
}
300+
log.Logger.Infof("Deleted key pair: %s", keyName)
301+
302+
return nil
303+
}

0 commit comments

Comments
 (0)