Skip to content

Commit ae1c70d

Browse files
hzhao-githubhuizhao
hzhao-github
and
huizhao
authored
Added FMW Integration test with Dynamic Cluster using WLST (#2246)
* Add FMW Integration test with Dynamic Cluster using WLST Co-authored-by: huizhao <[email protected]>
1 parent 21f3bc4 commit ae1c70d

File tree

2 files changed

+614
-0
lines changed

2 files changed

+614
-0
lines changed

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java

Lines changed: 363 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
// Copyright (c) 2021, Oracle and/or its affiliates.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
4+
package oracle.weblogic.kubernetes;
5+
6+
import java.io.File;
7+
import java.io.FileOutputStream;
8+
import java.nio.file.Path;
9+
import java.nio.file.Paths;
10+
import java.util.ArrayList;
11+
import java.util.Arrays;
12+
import java.util.List;
13+
import java.util.Properties;
14+
15+
import io.kubernetes.client.openapi.models.V1Container;
16+
import io.kubernetes.client.openapi.models.V1EnvVar;
17+
import io.kubernetes.client.openapi.models.V1LocalObjectReference;
18+
import io.kubernetes.client.openapi.models.V1ObjectMeta;
19+
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource;
20+
import io.kubernetes.client.openapi.models.V1SecretReference;
21+
import io.kubernetes.client.openapi.models.V1Volume;
22+
import io.kubernetes.client.openapi.models.V1VolumeMount;
23+
import oracle.weblogic.domain.AdminServer;
24+
import oracle.weblogic.domain.AdminService;
25+
import oracle.weblogic.domain.Channel;
26+
import oracle.weblogic.domain.Cluster;
27+
import oracle.weblogic.domain.Domain;
28+
import oracle.weblogic.domain.DomainSpec;
29+
import oracle.weblogic.domain.ServerPod;
30+
import oracle.weblogic.kubernetes.annotations.IntegrationTest;
31+
import oracle.weblogic.kubernetes.annotations.Namespaces;
32+
import oracle.weblogic.kubernetes.logging.LoggingFacade;
33+
import oracle.weblogic.kubernetes.utils.CommonTestUtils;
34+
import org.awaitility.core.ConditionFactory;
35+
import org.junit.jupiter.api.BeforeAll;
36+
import org.junit.jupiter.api.DisplayName;
37+
import org.junit.jupiter.api.Test;
38+
39+
import static java.util.concurrent.TimeUnit.MINUTES;
40+
import static java.util.concurrent.TimeUnit.SECONDS;
41+
import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
42+
import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT;
43+
import static oracle.weblogic.kubernetes.TestConstants.BASE_IMAGES_REPO_SECRET;
44+
import static oracle.weblogic.kubernetes.TestConstants.DB_IMAGE_TO_USE_IN_SPEC;
45+
import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION;
46+
import static oracle.weblogic.kubernetes.TestConstants.FMWINFRA_IMAGE_TO_USE_IN_SPEC;
47+
import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST;
48+
import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR;
49+
import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
50+
import static oracle.weblogic.kubernetes.actions.impl.primitive.Docker.getImageEnvVar;
51+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
52+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify;
53+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretForBaseImages;
54+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword;
55+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getExternalServicePodName;
56+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator;
57+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.setPodAntiAffinity;
58+
import static oracle.weblogic.kubernetes.utils.DbUtils.setupDBandRCUschema;
59+
import static oracle.weblogic.kubernetes.utils.TestUtils.callWebAppAndWaitTillReady;
60+
import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort;
61+
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
62+
import static org.awaitility.Awaitility.with;
63+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
64+
import static org.junit.jupiter.api.Assertions.assertNotNull;
65+
import static org.junit.jupiter.api.Assertions.assertTrue;
66+
67+
/**
68+
* Test to creat a FMW dynamic domain in persistent volume using WLST.
69+
*/
70+
@DisplayName("Test to creat a FMW dynamic domain in persistent volume using WLST")
71+
@IntegrationTest
72+
public class ItFmwDynamicDomainInPV {
73+
74+
private static ConditionFactory withStandardRetryPolicy;
75+
76+
private static String opNamespace = null;
77+
private static String domainNamespace = null;
78+
private static String dbNamespace = null;
79+
private static String oracle_home = null;
80+
private static String java_home = null;
81+
82+
private static final String RCUSCHEMAPREFIX = "fmwdomainpv";
83+
private static final String ORACLEDBURLPREFIX = "oracledb.";
84+
private static final String ORACLEDBSUFFIX = ".svc.cluster.local:1521/devpdb.k8s";
85+
private static final String RCUSYSUSERNAME = "sys";
86+
private static final String RCUSYSPASSWORD = "Oradoc_db1";
87+
private static final String RCUSCHEMAUSERNAME = "myrcuuser";
88+
private static final String RCUSCHEMAPASSWORD = "Oradoc_db1";
89+
90+
private static String dbUrl = null;
91+
private static LoggingFacade logger = null;
92+
93+
private static final String domainUid = "fmw-dynamicdomain-inpv";
94+
private static final String clusterName = "cluster-dynamicdomain-inpv";
95+
private static final String adminServerName = "admin-server";
96+
private static final String managedServerNameBase = "managed-server";
97+
private static final String adminServerPodName = domainUid + "-" + adminServerName;
98+
private static final String managedServerPodNamePrefix = domainUid + "-" + managedServerNameBase;
99+
private final int managedServerPort = 8001;
100+
private final String wlSecretName = domainUid + "-weblogic-credentials";
101+
private final String rcuSecretName = domainUid + "-rcu-credentials";
102+
private static final int replicaCount = 2;
103+
104+
/**
105+
* Assigns unique namespaces for DB, operator and domains.
106+
* Start DB service and create RCU schema.
107+
* Pull FMW image and Oracle DB image if running tests in Kind cluster.
108+
*/
109+
@BeforeAll
110+
public static void initAll(@Namespaces(3) List<String> namespaces) {
111+
logger = getLogger();
112+
// create standard, reusable retry/backoff policy
113+
withStandardRetryPolicy = with().pollDelay(10, SECONDS)
114+
.and().with().pollInterval(10, SECONDS)
115+
.atMost(5, MINUTES).await();
116+
117+
logger.info("Assign a unique namespace for DB and RCU");
118+
assertNotNull(namespaces.get(0), "Namespace is null");
119+
dbNamespace = namespaces.get(0);
120+
dbUrl = ORACLEDBURLPREFIX + dbNamespace + ORACLEDBSUFFIX;
121+
122+
logger.info("Assign a unique namespace for operator1");
123+
assertNotNull(namespaces.get(1), "Namespace is null");
124+
opNamespace = namespaces.get(1);
125+
126+
logger.info("Assign a unique namespace for FMW domain");
127+
assertNotNull(namespaces.get(2), "Namespace is null");
128+
domainNamespace = namespaces.get(2);
129+
130+
logger.info("Start DB and create RCU schema for namespace: {0}, RCU prefix: {1}, "
131+
+ "dbUrl: {2}, dbImage: {3}, fmwImage: {4} ", dbNamespace, RCUSCHEMAPREFIX, dbUrl,
132+
DB_IMAGE_TO_USE_IN_SPEC, FMWINFRA_IMAGE_TO_USE_IN_SPEC);
133+
assertDoesNotThrow(() -> setupDBandRCUschema(DB_IMAGE_TO_USE_IN_SPEC, FMWINFRA_IMAGE_TO_USE_IN_SPEC,
134+
RCUSCHEMAPREFIX, dbNamespace, 0, dbUrl),
135+
String.format("Failed to create RCU schema for prefix %s in the namespace %s with "
136+
+ "dbUrl %s", RCUSCHEMAPREFIX, dbNamespace, dbUrl));
137+
138+
logger.info("DB image: {0}, FMW image {1} used in the test",
139+
DB_IMAGE_TO_USE_IN_SPEC, FMWINFRA_IMAGE_TO_USE_IN_SPEC);
140+
141+
// install operator and verify its running in ready state
142+
installAndVerifyOperator(opNamespace, domainNamespace);
143+
}
144+
145+
/**
146+
* Create a basic FMW dynamic cluster model in image domain.
147+
* Verify Pod is ready and service exists for both admin server and managed servers.
148+
* Verify EM console is accessible.
149+
*/
150+
@Test
151+
@DisplayName("Create FMW Dynamic Domain in PV")
152+
public void testFmwDynamicDomainInPV() {
153+
// create FMW dynamic domain and verify
154+
createFmwDomainAndVerify();
155+
verifyDomainReady();
156+
}
157+
158+
private void createFmwDomainAndVerify() {
159+
final String pvName = domainUid + "-" + domainNamespace + "-pv";
160+
final String pvcName = domainUid + "-" + domainNamespace + "-pvc";
161+
final int t3ChannelPort = getNextFreePort(30000, 32767);
162+
163+
// create pull secrets for domainNamespace when running in non Kind Kubernetes cluster
164+
// this secret is used only for non-kind cluster
165+
createSecretForBaseImages(domainNamespace);
166+
167+
// create FMW domain credential secret
168+
createSecretWithUsernamePassword(wlSecretName, domainNamespace,
169+
ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT);
170+
171+
// create RCU credential secret
172+
CommonTestUtils.createRcuSecretWithUsernamePassword(rcuSecretName, domainNamespace,
173+
RCUSCHEMAUSERNAME, RCUSCHEMAPASSWORD, RCUSYSUSERNAME, RCUSYSPASSWORD);
174+
175+
// create persistent volume and persistent volume claim for domain
176+
CommonTestUtils.createPV(pvName, domainUid, this.getClass().getSimpleName());
177+
CommonTestUtils.createPVC(pvName, pvcName, domainUid, domainNamespace);
178+
179+
File domainPropertiesFile = createWlstPropertyFile(t3ChannelPort);
180+
181+
// WLST script for creating domain
182+
Path wlstScript = Paths.get(RESOURCE_DIR, "python-scripts", "fmw-create-dynamic-domain.py");
183+
184+
// create configmap and domain on persistent volume using the WLST script and property file
185+
createDomainOnPvUsingWlst(wlstScript, domainPropertiesFile.toPath(), pvName, pvcName);
186+
187+
// create domain and verify
188+
createDomainCrAndVerify(pvName, pvcName, t3ChannelPort);
189+
}
190+
191+
private void createDomainCrAndVerify(String pvName,
192+
String pvcName,
193+
int t3ChannelPort) {
194+
// create a domain custom resource configuration object
195+
logger.info("Creating domain custom resource");
196+
Domain domain = new Domain()
197+
.apiVersion(DOMAIN_API_VERSION)
198+
.kind("Domain")
199+
.metadata(new V1ObjectMeta()
200+
.name(domainUid)
201+
.namespace(domainNamespace))
202+
.spec(new DomainSpec()
203+
.domainUid(domainUid)
204+
.domainHome("/shared/domains/" + domainUid) // point to domain home in pv
205+
.domainHomeSourceType("PersistentVolume") // set the domain home source type as pv
206+
.image(FMWINFRA_IMAGE_TO_USE_IN_SPEC)
207+
.imagePullPolicy("IfNotPresent")
208+
.imagePullSecrets(Arrays.asList(
209+
new V1LocalObjectReference()
210+
.name(BASE_IMAGES_REPO_SECRET)))
211+
.webLogicCredentialsSecret(new V1SecretReference()
212+
.name(wlSecretName)
213+
.namespace(domainNamespace))
214+
.includeServerOutInPodLog(true)
215+
.logHomeEnabled(Boolean.TRUE)
216+
.logHome("/shared/logs/" + domainUid)
217+
.dataHome("")
218+
.serverStartPolicy("IF_NEEDED")
219+
.serverPod(new ServerPod() //serverpod
220+
.addEnvItem(new V1EnvVar()
221+
.name("JAVA_OPTIONS")
222+
.value("-Dweblogic.StdoutDebugEnabled=false"))
223+
.addEnvItem(new V1EnvVar()
224+
.name("USER_MEM_ARGS")
225+
.value("-Djava.security.egd=file:/dev/./urandom"))
226+
.addEnvItem(new V1EnvVar()
227+
.name("ALLOW_DYNAMIC_CLUSTER_IN_FMW")
228+
.value("true"))
229+
.addVolumesItem(new V1Volume()
230+
.name(pvName)
231+
.persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource()
232+
.claimName(pvcName)))
233+
.addVolumeMountsItem(new V1VolumeMount()
234+
.mountPath("/shared")
235+
.name(pvName)))
236+
.adminServer(new AdminServer() //admin server
237+
.serverStartState("RUNNING")
238+
.adminService(new AdminService()
239+
.addChannelsItem(new Channel()
240+
.channelName("default")
241+
.nodePort(0))
242+
.addChannelsItem(new Channel()
243+
.channelName("T3Channel")
244+
.nodePort(t3ChannelPort))))
245+
.addClustersItem(new Cluster() //cluster
246+
.clusterName(clusterName)
247+
.replicas(replicaCount)
248+
.serverStartState("RUNNING")
249+
));
250+
setPodAntiAffinity(domain);
251+
252+
// verify the domain custom resource is created
253+
createDomainAndVerify(domain, domainNamespace);
254+
}
255+
256+
private void createDomainOnPvUsingWlst(Path wlstScriptFile,
257+
Path domainPropertiesFile,
258+
String pvName,
259+
String pvcName) {
260+
261+
logger.info("Preparing to run create domain job using WLST");
262+
263+
List<Path> domainScriptFiles = new ArrayList<>();
264+
domainScriptFiles.add(wlstScriptFile);
265+
domainScriptFiles.add(domainPropertiesFile);
266+
267+
logger.info("Creating a config map to hold domain creation scripts");
268+
String domainScriptConfigMapName = "create-domain-scripts-cm";
269+
assertDoesNotThrow(
270+
() -> CommonTestUtils.createConfigMapForDomainCreation(domainScriptConfigMapName, domainScriptFiles,
271+
domainNamespace, this.getClass().getSimpleName()),
272+
"Create configmap for domain creation failed");
273+
274+
// create a V1Container with specific scripts and properties for creating domain
275+
V1Container jobCreationContainer = new V1Container()
276+
.addCommandItem("/bin/sh")
277+
.addArgsItem("/u01/oracle/oracle_common/common/bin/wlst.sh")
278+
.addArgsItem("/u01/weblogic/" + wlstScriptFile.getFileName()) //wlst.sh script
279+
.addArgsItem("-skipWLSModuleScanning")
280+
.addArgsItem("-loadProperties")
281+
.addArgsItem("/u01/weblogic/" + domainPropertiesFile.getFileName()); //domain property file
282+
283+
logger.info("Running a Kubernetes job to create the domain");
284+
CommonTestUtils.createDomainJob(FMWINFRA_IMAGE_TO_USE_IN_SPEC, pvName, pvcName, domainScriptConfigMapName,
285+
domainNamespace, jobCreationContainer);
286+
}
287+
288+
private File createWlstPropertyFile(int t3ChannelPort) {
289+
//get ENV variable from the image
290+
assertNotNull(getImageEnvVar(FMWINFRA_IMAGE_TO_USE_IN_SPEC, "ORACLE_HOME"),
291+
"envVar ORACLE_HOME from image is null");
292+
oracle_home = getImageEnvVar(FMWINFRA_IMAGE_TO_USE_IN_SPEC, "ORACLE_HOME");
293+
logger.info("ORACLE_HOME in image {0} is: {1}", FMWINFRA_IMAGE_TO_USE_IN_SPEC, oracle_home);
294+
assertNotNull(getImageEnvVar(FMWINFRA_IMAGE_TO_USE_IN_SPEC, "JAVA_HOME"),
295+
"envVar JAVA_HOME from image is null");
296+
java_home = getImageEnvVar(FMWINFRA_IMAGE_TO_USE_IN_SPEC, "JAVA_HOME");
297+
logger.info("JAVA_HOME in image {0} is: {1}", FMWINFRA_IMAGE_TO_USE_IN_SPEC, java_home);
298+
299+
// create wlst property file object
300+
Properties p = new Properties();
301+
p.setProperty("oracleHome", oracle_home); //default $ORACLE_HOME
302+
p.setProperty("javaHome", java_home); //default $JAVA_HOME
303+
p.setProperty("domainParentDir", "/shared/domains/");
304+
p.setProperty("domainName", domainUid);
305+
p.setProperty("domainUser", ADMIN_USERNAME_DEFAULT);
306+
p.setProperty("domainPassword", ADMIN_PASSWORD_DEFAULT);
307+
p.setProperty("rcuDb", dbUrl);
308+
p.setProperty("rcuSchemaPrefix", RCUSCHEMAPREFIX);
309+
p.setProperty("rcuSchemaPassword", RCUSCHEMAPASSWORD);
310+
p.setProperty("adminListenPort", "7001");
311+
p.setProperty("adminName", adminServerName);
312+
p.setProperty("adminPodName", adminServerPodName);
313+
p.setProperty("adminUsername", ADMIN_USERNAME_DEFAULT);
314+
p.setProperty("adminPassword", ADMIN_PASSWORD_DEFAULT);
315+
p.setProperty("managedNameBase", managedServerNameBase);
316+
p.setProperty("managedServerPort", Integer.toString(managedServerPort));
317+
p.setProperty("prodMode", "true");
318+
p.setProperty("managedCount", "4");
319+
p.setProperty("clusterName", clusterName);
320+
p.setProperty("t3ChannelPublicAddress", K8S_NODEPORT_HOST);
321+
p.setProperty("t3ChannelPort", Integer.toString(t3ChannelPort));
322+
p.setProperty("exposeAdminT3Channel", "true");
323+
324+
// create a temporary WebLogic domain property file
325+
File domainPropertiesFile = assertDoesNotThrow(() ->
326+
File.createTempFile("domain", "properties"),
327+
"Failed to create domain properties file");
328+
329+
// create the property file
330+
assertDoesNotThrow(() ->
331+
p.store(new FileOutputStream(domainPropertiesFile), "FMW wlst properties file"),
332+
"Failed to write domain properties file");
333+
334+
return domainPropertiesFile;
335+
}
336+
337+
/**
338+
* Verify Pod is ready and service exists for both admin server and managed servers.
339+
* Verify EM console is accessible.
340+
*/
341+
private void verifyDomainReady() {
342+
checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace);
343+
for (int i = 1; i <= replicaCount; i++) {
344+
logger.info("Checking managed server service {0} is created in namespace {1}",
345+
managedServerPodNamePrefix + i, domainNamespace);
346+
checkPodReadyAndServiceExists(managedServerPodNamePrefix + i, domainUid, domainNamespace);
347+
}
348+
349+
//check access to the em console: http://hostname:port/em
350+
int nodePort = getServiceNodePort(
351+
domainNamespace, getExternalServicePodName(adminServerPodName), "default");
352+
assertTrue(nodePort != -1,
353+
"Could not get the default external service node port");
354+
logger.info("Found the default service nodePort {0}", nodePort);
355+
String curlCmd1 = "curl -s -L --show-error --noproxy '*' "
356+
+ " http://" + K8S_NODEPORT_HOST + ":" + nodePort
357+
+ "/em --write-out %{http_code} -o /dev/null";
358+
logger.info("Executing default nodeport curl command {0}", curlCmd1);
359+
assertTrue(callWebAppAndWaitTillReady(curlCmd1, 5), "Calling web app failed");
360+
logger.info("EM console is accessible thru default service");
361+
}
362+
}
363+

0 commit comments

Comments
 (0)