Skip to content

Commit 4170b5f

Browse files
michaelwittigmichaelwittig
committed
make tests more stable: support retain on failure
1 parent fd80770 commit 4170b5f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+542
-380
lines changed

‎test/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ To run this tests you need:
2525
* `BUCKET_NAME` Some templates are to big to be passed as a string from local disk, therefore you need to supply the name of the bucket that is used to upload templates.
2626
* `BUCKET_REGION` **required if BUCKET_NAME is set** Region of the bucket
2727
* `DELETION_POLICY` (default `delete`, allowed values [`delete`, `retain`]) should resources be deleted?
28+
* `FAILURE_POLICY` (default `rollback`, allowed values [`rollback`, `retain`]) what happens if a stack fails?
2829

2930
## Usage
3031

‎test/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@
7070
<dependency>
7171
<groupId>com.amazonaws</groupId>
7272
<artifactId>aws-java-sdk-bom</artifactId>
73-
<version>1.11.133</version>
73+
<version>1.11.837</version>
7474
<type>pom</type>
7575
<scope>import</scope>
7676
</dependency>

‎test/src/test/java/de/widdix/awscftemplates/AAWSTest.java

Lines changed: 48 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,15 @@ protected final User createUser(final String userName) throws JSchException {
6767
return new User(userName, sshPrivateKeyBlob, res.getSSHPublicKey().getSSHPublicKeyId());
6868
}
6969

70-
protected final void deleteUser(final String userName) {
70+
protected final void deleteUser(final Context context, final String userName) {
7171
if (Config.get(Config.Key.DELETION_POLICY).equals("delete")) {
72-
final ListSSHPublicKeysResult res = this.iam.listSSHPublicKeys(new ListSSHPublicKeysRequest().withUserName(userName));
73-
this.iam.deleteSSHPublicKey(new DeleteSSHPublicKeyRequest().withUserName(userName).withSSHPublicKeyId(res.getSSHPublicKeys().get(0).getSSHPublicKeyId()));
74-
this.iam.deleteUser(new DeleteUserRequest().withUserName(userName));
72+
if (Config.get(Config.Key.FAILURE_POLICY).equals("retain") && context.hasStackFailure()) {
73+
System.out.println("Skip user deletion because of stack failure in context and FAILURE_POLICY := retain");
74+
} else {
75+
final ListSSHPublicKeysResult res = this.iam.listSSHPublicKeys(new ListSSHPublicKeysRequest().withUserName(userName));
76+
this.iam.deleteSSHPublicKey(new DeleteSSHPublicKeyRequest().withUserName(userName).withSSHPublicKeyId(res.getSSHPublicKeys().get(0).getSSHPublicKeyId()));
77+
this.iam.deleteUser(new DeleteUserRequest().withUserName(userName));
78+
}
7579
}
7680
}
7781

@@ -81,10 +85,14 @@ protected final KeyPair createKey(final String keyName) {
8185
return res.getKeyPair();
8286
}
8387

84-
protected final void deleteKey(final String keyName) {
88+
protected final void deleteKey(final Context context, final String keyName) {
8589
if (Config.get(Config.Key.DELETION_POLICY).equals("delete")) {
86-
this.ec2.deleteKeyPair(new DeleteKeyPairRequest().withKeyName(keyName));
87-
System.out.println("keypair[" + keyName + "] deleted");
90+
if (Config.get(Config.Key.FAILURE_POLICY).equals("retain") && context.hasStackFailure()) {
91+
System.out.println("Skip key deletion because of stack failure in context and FAILURE_POLICY := retain");
92+
} else {
93+
this.ec2.deleteKeyPair(new DeleteKeyPairRequest().withKeyName(keyName));
94+
System.out.println("keypair[" + keyName + "] deleted");
95+
}
8896
}
8997
}
9098

@@ -97,39 +105,51 @@ protected final void createObject(final String bucketName, final String key, fin
97105
this.s3.putObject(bucketName, key, body);
98106
}
99107

100-
protected final void deleteObject(final String bucketName, final String key) {
108+
protected final void deleteObject(final Context context, final String bucketName, final String key) {
101109
if (Config.get(Config.Key.DELETION_POLICY).equals("delete")) {
102-
this.s3.deleteObject(bucketName, key);
110+
if (Config.get(Config.Key.FAILURE_POLICY).equals("retain") && context.hasStackFailure()) {
111+
System.out.println("Skip object deletion because of stack failure in context and FAILURE_POLICY := retain");
112+
} else {
113+
this.s3.deleteObject(bucketName, key);
114+
}
103115
}
104116
}
105117

106-
protected final void emptyBucket(final String name) {
118+
protected final void emptyBucket(final Context context, final String name) {
107119
if (Config.get(Config.Key.DELETION_POLICY).equals("delete")) {
108-
ObjectListing objectListing = s3.listObjects(name);
109-
while (true) {
110-
objectListing.getObjectSummaries().forEach((summary) -> s3.deleteObject(name, summary.getKey()));
111-
if (objectListing.isTruncated()) {
112-
objectListing = s3.listNextBatchOfObjects(objectListing);
113-
} else {
114-
break;
120+
if (Config.get(Config.Key.FAILURE_POLICY).equals("retain") && context.hasStackFailure()) {
121+
System.out.println("Skip bucket empty because of stack failure in context and FAILURE_POLICY := retain");
122+
} else {
123+
ObjectListing objectListing = s3.listObjects(name);
124+
while (true) {
125+
objectListing.getObjectSummaries().forEach((summary) -> s3.deleteObject(name, summary.getKey()));
126+
if (objectListing.isTruncated()) {
127+
objectListing = s3.listNextBatchOfObjects(objectListing);
128+
} else {
129+
break;
130+
}
115131
}
116-
}
117-
VersionListing versionListing = s3.listVersions(new ListVersionsRequest().withBucketName(name));
118-
while (true) {
119-
versionListing.getVersionSummaries().forEach((vs) -> s3.deleteVersion(name, vs.getKey(), vs.getVersionId()));
120-
if (versionListing.isTruncated()) {
121-
versionListing = s3.listNextBatchOfVersions(versionListing);
122-
} else {
123-
break;
132+
VersionListing versionListing = s3.listVersions(new ListVersionsRequest().withBucketName(name));
133+
while (true) {
134+
versionListing.getVersionSummaries().forEach((vs) -> s3.deleteVersion(name, vs.getKey(), vs.getVersionId()));
135+
if (versionListing.isTruncated()) {
136+
versionListing = s3.listNextBatchOfVersions(versionListing);
137+
} else {
138+
break;
139+
}
124140
}
125141
}
126142
}
127143
}
128144

129-
protected final void deleteBucket(final String name) {
145+
protected final void deleteBucket(final Context context, final String name) {
130146
if (Config.get(Config.Key.DELETION_POLICY).equals("delete")) {
131-
this.emptyBucket(name);
132-
this.s3.deleteBucket(new DeleteBucketRequest(name));
147+
if (Config.get(Config.Key.FAILURE_POLICY).equals("retain") && context.hasStackFailure()) {
148+
System.out.println("Skip bucket deletion because of stack failure in context and FAILURE_POLICY := retain");
149+
} else {
150+
this.emptyBucket(context, name);
151+
this.s3.deleteBucket(new DeleteBucketRequest(name));
152+
}
133153
}
134154
}
135155

‎test/src/test/java/de/widdix/awscftemplates/ACloudFormationTest.java

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import java.io.File;
1111
import java.io.IOException;
1212
import java.nio.charset.Charset;
13+
import java.nio.charset.StandardCharsets;
1314
import java.nio.file.Files;
1415
import java.nio.file.Paths;
1516
import java.util.*;
@@ -32,7 +33,8 @@ public ACloudFormationTest() {
3233
super();
3334
}
3435

35-
protected final void createStack(final String stackName, final String template, final Parameter... parameters) {
36+
protected final void createStack(final Context context, final String stackName, final String template, final Parameter... parameters) {
37+
context.addStack(stackName);
3638
CreateStackRequest req = new CreateStackRequest()
3739
.withStackName(stackName)
3840
.withParameters(parameters)
@@ -46,17 +48,20 @@ protected final void createStack(final String stackName, final String template,
4648
s3local.putObject(bucketName, stackName, new File(dir + template));
4749
req = req.withTemplateURL("https://s3-" + bucketRegion + ".amazonaws.com/" + bucketName + "/" + stackName);
4850
} else {
49-
final String body = readFile(dir + template, Charset.forName("UTF-8"));
51+
final String body = readFile(dir + template, StandardCharsets.UTF_8);
5052
req = req.withTemplateBody(body);
5153
}
5254
} else {
5355
req = req.withTemplateURL("https://s3-eu-west-1.amazonaws.com/widdix-aws-cf-templates/" + template);
5456
}
57+
if (Config.get(Config.Key.FAILURE_POLICY).equals("retain")) {
58+
req = req.withOnFailure(OnFailure.DO_NOTHING);
59+
}
5560
this.cf.createStack(req);
56-
this.waitForStack(stackName, FinalStatus.CREATE_COMPLETE);
61+
this.waitForStack(context, stackName, FinalStatus.CREATE_COMPLETE);
5762
}
5863

59-
protected final void updateStack(final String stackName, final String template, final Parameter... parameters) {
64+
protected final void updateStack(final Context context, final String stackName, final String template, final Parameter... parameters) {
6065
UpdateStackRequest req = new UpdateStackRequest()
6166
.withStackName(stackName)
6267
.withParameters(parameters)
@@ -70,14 +75,14 @@ protected final void updateStack(final String stackName, final String template,
7075
s3local.putObject(bucketName, stackName, new File(dir + template));
7176
req = req.withTemplateURL("https://s3-" + bucketRegion + ".amazonaws.com/" + bucketName + "/" + stackName);
7277
} else {
73-
final String body = readFile(dir + template, Charset.forName("UTF-8"));
78+
final String body = readFile(dir + template, StandardCharsets.UTF_8);
7479
req = req.withTemplateBody(body);
7580
}
7681
} else {
7782
req = req.withTemplateURL("https://s3-eu-west-1.amazonaws.com/widdix-aws-cf-templates/" + template);
7883
}
7984
this.cf.updateStack(req);
80-
this.waitForStack(stackName, FinalStatus.UPDATE_COMPLETE);
85+
this.waitForStack(context, stackName, FinalStatus.UPDATE_COMPLETE);
8186
}
8287

8388
protected enum FinalStatus {
@@ -118,7 +123,7 @@ private List<StackEvent> getStackEvents(final String stackName) {
118123
return events;
119124
}
120125

121-
private void waitForStack(final String stackName, final FinalStatus finalStackStatus) {
126+
private void waitForStack(final Context context, final String stackName, final FinalStatus finalStackStatus) {
122127
System.out.println("waitForStack[" + stackName + "]: to reach status " + finalStackStatus.finalStatus);
123128
final List<StackEvent> eventsDisplayed = new ArrayList<>();
124129
while (true) {
@@ -150,6 +155,7 @@ private void waitForStack(final String stackName, final FinalStatus finalStackSt
150155
if (finalStackStatus.intermediateStatus.contains(currentStatus)) {
151156
System.out.println("waitForStack[" + stackName + "]: continue to wait (still in intermediate status " + currentStatus + ") ...");
152157
} else {
158+
context.reportStackFailure(stackName);
153159
throw new RuntimeException("waitForStack[" + stackName + "]: reached invalid intermediate status " + currentStatus + ".");
154160
}
155161
}
@@ -162,6 +168,7 @@ private void waitForStack(final String stackName, final FinalStatus finalStackSt
162168
if (finalStackStatus.notFoundIsIntermediateStatus) {
163169
System.out.println("waitForStack[" + stackName + "]: continue to wait (stack not found) ...");
164170
} else {
171+
context.reportStackFailure(stackName);
165172
throw new RuntimeException("waitForStack[" + stackName + "]: stack not found.");
166173
}
167174
}
@@ -186,22 +193,26 @@ protected final String getStackOutputValue(final String stackName, final String
186193
return this.getStackOutputs(stackName).get(outputKey);
187194
}
188195

189-
protected final void deleteStackAndRetryOnFailure(final String stackName) {
196+
protected final void deleteStackAndRetryOnFailure(final Context context, final String stackName) {
190197
final Callable<Boolean> callable = () -> {
191-
this.deleteStack(stackName);
198+
this.deleteStack(context, stackName);
192199
return true;
193200
};
194201
this.retry(callable);
195202
}
196203

197-
protected final void deleteStack(final String stackName) {
204+
protected final void deleteStack(final Context context, final String stackName) {
198205
if (Config.get(Config.Key.DELETION_POLICY).equals("delete")) {
199-
this.cf.deleteStack(new DeleteStackRequest().withStackName(stackName));
206+
if (Config.get(Config.Key.FAILURE_POLICY).equals("retain") && context.hasStackFailure()) {
207+
System.out.println("Skip stack deletion because of stack failure in context and FAILURE_POLICY := retain");
208+
} else {
209+
this.cf.deleteStack(new DeleteStackRequest().withStackName(stackName));
210+
}
200211
if (Config.has(Config.Key.BUCKET_NAME)) {
201212
final AmazonS3 s3local = AmazonS3ClientBuilder.standard().withCredentials(this.credentialsProvider).withRegion(Config.get(Config.Key.BUCKET_REGION)).build();
202213
s3local.deleteObject(Config.get(Config.Key.BUCKET_NAME), stackName);
203214
}
204-
this.waitForStack(stackName, FinalStatus.DELETE_COMPLETE);
215+
this.waitForStack(context, stackName, FinalStatus.DELETE_COMPLETE);
205216
}
206217
}
207218

‎test/src/test/java/de/widdix/awscftemplates/Config.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ public enum Key {
1111
DOMAIN_SUFFIX("DOMAIN_SUFFIX"),
1212
ACM_CERTIFICATE_ARN("ACM_CERTIFICATE_ARN"),
1313
CLOUDFRONT_ACM_CERTIFICATE_ARN("CLOUDFRONT_ACM_CERTIFICATE_ARN"),
14-
DELETION_POLICY("DELETION_POLICY", "delete");
14+
DELETION_POLICY("DELETION_POLICY", "delete"),
15+
FAILURE_POLICY("FAILURE_POLICY", "rollback");
1516

1617
private final String name;
1718
private final String defaultValue;

‎test/src/test/java/de/widdix/awscftemplates/Context.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package de.widdix.awscftemplates;
2+
3+
import java.util.concurrent.ConcurrentHashMap;
4+
5+
public class Context {
6+
7+
private final ConcurrentHashMap<String, Boolean> stacks = new ConcurrentHashMap<>(8);
8+
9+
private volatile boolean failure = false;
10+
11+
public void addStack(final String stackName) {
12+
stacks.put(stackName, true);
13+
}
14+
public void reportStackFailure(final String stackName) {
15+
if (!stacks.containsKey(stackName)) {
16+
throw new RuntimeException("stack not in context");
17+
}
18+
this.failure = true;
19+
}
20+
21+
public boolean hasStackFailure() {
22+
return this.failure;
23+
}
24+
}

‎test/src/test/java/de/widdix/awscftemplates/ec2/TestAL2MutablePrivate.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,41 +2,43 @@
22

33
import com.amazonaws.services.cloudformation.model.Parameter;
44
import de.widdix.awscftemplates.ACloudFormationTest;
5+
import de.widdix.awscftemplates.Context;
56
import org.junit.Test;
67

78
public class TestAL2MutablePrivate extends ACloudFormationTest {
89

910
@Test
1011
public void test() {
12+
final Context context = new Context();
1113
final String vpcStackName = "vpc-2azs-" + this.random8String();
1214
final String natStackName = "vpc-nat-gateway-" + this.random8String();
1315
final String stackName = "al2-mutable-private-" + this.random8String();
1416
final String classB = "10";
1517
try {
16-
this.createStack(vpcStackName,
18+
this.createStack(context, vpcStackName,
1719
"vpc/vpc-2azs.yaml",
1820
new Parameter().withParameterKey("ClassB").withParameterValue(classB)
1921
);
2022
try {
21-
this.createStack(natStackName,
23+
this.createStack(context, natStackName,
2224
"vpc/vpc-nat-gateway.yaml",
2325
new Parameter().withParameterKey("ParentVPCStack").withParameterValue(vpcStackName)
2426
);
2527
try {
26-
this.createStack(stackName,
28+
this.createStack(context, stackName,
2729
"ec2/al2-mutable-private.yaml",
2830
new Parameter().withParameterKey("ParentVPCStack").withParameterValue(vpcStackName),
2931
new Parameter().withParameterKey("BackupRetentionPeriod").withParameterValue("0")
3032
);
3133
// TODO how can we check if this stack works?
3234
} finally {
33-
this.deleteStack(stackName);
35+
this.deleteStack(context, stackName);
3436
}
3537
} finally {
36-
this.deleteStack(natStackName);
38+
this.deleteStack(context, natStackName);
3739
}
3840
} finally {
39-
this.deleteStack(vpcStackName);
41+
this.deleteStack(context, vpcStackName);
4042
}
4143
}
4244

0 commit comments

Comments
 (0)