1
1
package com .devshawn .kafka .gitops ;
2
2
3
- import ch .qos .logback .classic .Level ;
4
- import ch .qos .logback .classic .Logger ;
3
+ import java .util .ArrayList ;
4
+ import java .util .List ;
5
+ import java .util .Map ;
6
+ import java .util .NoSuchElementException ;
7
+ import java .util .Optional ;
8
+ import java .util .concurrent .atomic .AtomicInteger ;
9
+ import java .util .concurrent .atomic .AtomicReference ;
10
+ import org .slf4j .LoggerFactory ;
5
11
import com .devshawn .kafka .gitops .config .KafkaGitopsConfigLoader ;
6
12
import com .devshawn .kafka .gitops .config .ManagerConfig ;
13
+ import com .devshawn .kafka .gitops .config .SchemaRegistryConfigLoader ;
7
14
import com .devshawn .kafka .gitops .domain .confluent .ServiceAccount ;
8
15
import com .devshawn .kafka .gitops .domain .options .GetAclOptions ;
9
16
import com .devshawn .kafka .gitops .domain .plan .DesiredPlan ;
10
17
import com .devshawn .kafka .gitops .domain .state .AclDetails ;
11
18
import com .devshawn .kafka .gitops .domain .state .CustomAclDetails ;
12
19
import com .devshawn .kafka .gitops .domain .state .DesiredState ;
13
20
import com .devshawn .kafka .gitops .domain .state .DesiredStateFile ;
21
+ import com .devshawn .kafka .gitops .domain .state .SchemaDetails ;
14
22
import com .devshawn .kafka .gitops .domain .state .TopicDetails ;
15
23
import com .devshawn .kafka .gitops .domain .state .service .KafkaStreamsService ;
16
- import com .devshawn .kafka .gitops .enums .SchemaType ;
24
+ import com .devshawn .kafka .gitops .enums .SchemaCompatibility ;
17
25
import com .devshawn .kafka .gitops .exception .ConfluentCloudException ;
18
26
import com .devshawn .kafka .gitops .exception .InvalidAclDefinitionException ;
19
27
import com .devshawn .kafka .gitops .exception .MissingConfigurationException ;
25
33
import com .devshawn .kafka .gitops .service .KafkaService ;
26
34
import com .devshawn .kafka .gitops .service .ParserService ;
27
35
import com .devshawn .kafka .gitops .service .RoleService ;
28
- import com .devshawn .kafka .gitops .config .SchemaRegistryConfig ;
29
- import com .devshawn .kafka .gitops .config .SchemaRegistryConfigLoader ;
30
36
import com .devshawn .kafka .gitops .service .SchemaRegistryService ;
31
37
import com .devshawn .kafka .gitops .util .LogUtil ;
32
38
import com .devshawn .kafka .gitops .util .StateUtil ;
33
39
import com .fasterxml .jackson .core .JsonParser ;
40
+ import com .fasterxml .jackson .core .util .DefaultIndenter ;
41
+ import com .fasterxml .jackson .core .util .DefaultPrettyPrinter ;
34
42
import com .fasterxml .jackson .databind .DeserializationFeature ;
35
43
import com .fasterxml .jackson .databind .ObjectMapper ;
44
+ import com .fasterxml .jackson .databind .SerializationFeature ;
36
45
import com .fasterxml .jackson .datatype .jdk8 .Jdk8Module ;
37
- import org .slf4j .LoggerFactory ;
38
-
39
- import java .nio .file .Files ;
40
- import java .nio .file .Paths ;
41
- import java .util .ArrayList ;
42
- import java .util .List ;
43
- import java .util .Map ;
44
- import java .util .NoSuchElementException ;
45
- import java .util .Optional ;
46
- import java .util .concurrent .atomic .AtomicInteger ;
47
- import java .util .concurrent .atomic .AtomicReference ;
46
+ import ch .qos .logback .classic .Level ;
47
+ import ch .qos .logback .classic .Logger ;
48
48
49
49
public class StateManager {
50
50
@@ -168,7 +168,7 @@ private DesiredState getDesiredState() {
168
168
}
169
169
170
170
private void generateTopicsState (DesiredState .Builder desiredState , DesiredStateFile desiredStateFile ) {
171
- Optional <Integer > defaultReplication = StateUtil .fetchReplication (desiredStateFile );
171
+ Optional <Integer > defaultReplication = StateUtil .fetchDefaultTopicsReplication (desiredStateFile );
172
172
if (defaultReplication .isPresent ()) {
173
173
desiredStateFile .getTopics ().forEach ((name , details ) -> {
174
174
Integer replication = details .getReplication ().isPresent () ? details .getReplication ().get () : defaultReplication .get ();
@@ -180,7 +180,15 @@ private void generateTopicsState(DesiredState.Builder desiredState, DesiredState
180
180
}
181
181
182
182
private void generateSchemasState (DesiredState .Builder desiredState , DesiredStateFile desiredStateFile ) {
183
- desiredState .putAllSchemas (desiredStateFile .getSchemas ());
183
+ Optional <SchemaCompatibility > defaultSchemaCompatibility = StateUtil .fetchDefaultSchemasCompatibility (desiredStateFile );
184
+ if (defaultSchemaCompatibility .isPresent ()) {
185
+ desiredStateFile .getSchemas ().forEach ((s , details ) -> {
186
+ SchemaCompatibility compatibility = details .getCompatibility ().isPresent () ? details .getCompatibility ().get () : defaultSchemaCompatibility .get ();
187
+ desiredState .putSchemas (s , new SchemaDetails .Builder ().mergeFrom (details ).setCompatibility (compatibility ).build ());
188
+ });
189
+ } else {
190
+ desiredState .putAllSchemas (desiredStateFile .getSchemas ());
191
+ }
184
192
}
185
193
186
194
private void generateConfluentCloudServiceAcls (DesiredState .Builder desiredState , DesiredStateFile desiredStateFile ) {
@@ -321,7 +329,7 @@ private void validateCustomAcls(DesiredStateFile desiredStateFile) {
321
329
}
322
330
323
331
private void validateTopics (DesiredStateFile desiredStateFile ) {
324
- Optional <Integer > defaultReplication = StateUtil .fetchReplication (desiredStateFile );
332
+ Optional <Integer > defaultReplication = StateUtil .fetchDefaultTopicsReplication (desiredStateFile );
325
333
if (!defaultReplication .isPresent ()) {
326
334
desiredStateFile .getTopics ().forEach ((name , details ) -> {
327
335
if (!details .getReplication ().isPresent ()) {
@@ -336,18 +344,13 @@ private void validateTopics(DesiredStateFile desiredStateFile) {
336
344
}
337
345
338
346
private void validateSchemas (DesiredStateFile desiredStateFile ) {
339
- if (!desiredStateFile .getSchemas ().isEmpty ()) {
340
- SchemaRegistryConfig schemaRegistryConfig = SchemaRegistryConfigLoader .load ();
341
- desiredStateFile .getSchemas ().forEach ((s , schemaDetails ) -> {
342
- if (!schemaDetails .getType ().equalsIgnoreCase (SchemaType .AVRO .toString ()) ||
343
- !schemaDetails .getType ().equalsIgnoreCase (SchemaType .JSON .toString ()) ||
344
- !schemaDetails .getType ().equalsIgnoreCase (SchemaType .PROTOBUF .toString ())) {
345
- throw new ValidationException (String .format ("Schema type %s is currently not supported." , schemaDetails .getType ()));
346
- }
347
- if (!Files .exists (Paths .get (schemaRegistryConfig .getConfig ().get ("SCHEMA_DIRECTORY" ) + "/" + schemaDetails .getFile ()))) {
348
- throw new ValidationException (String .format ("Schema file %s not found in schema directory at %s" , schemaDetails .getFile (), schemaRegistryConfig .getConfig ().get ("SCHEMA_DIRECTORY" )));
347
+ Optional <SchemaCompatibility > defaultSchemaCompatibility = StateUtil .fetchDefaultSchemasCompatibility (desiredStateFile );
348
+ if (!defaultSchemaCompatibility .isPresent ()) {
349
+ desiredStateFile .getSchemas ().forEach ((subject , details ) -> {
350
+ if (!details .getCompatibility ().isPresent ()) {
351
+ throw new ValidationException (String .format ("Not set: [compatibility] in state file definition: schema -> %s" , subject ));
349
352
}
350
- schemaRegistryService .validateSchema (schemaDetails );
353
+ schemaRegistryService .validateSchema (subject , details );
351
354
});
352
355
}
353
356
}
@@ -360,11 +363,17 @@ private boolean isConfluentCloudEnabled(DesiredStateFile desiredStateFile) {
360
363
}
361
364
362
365
private ObjectMapper initializeObjectMapper () {
363
- ObjectMapper objectMapper = new ObjectMapper ();
364
- objectMapper .enable (JsonParser .Feature .ALLOW_UNQUOTED_FIELD_NAMES );
365
- objectMapper .configure (DeserializationFeature .FAIL_ON_UNKNOWN_PROPERTIES , false );
366
- objectMapper .registerModule (new Jdk8Module ());
367
- return objectMapper ;
366
+ ObjectMapper gitopsObjectMapper = new ObjectMapper ();
367
+ gitopsObjectMapper .enable (SerializationFeature .INDENT_OUTPUT );
368
+ gitopsObjectMapper .enable (JsonParser .Feature .ALLOW_UNQUOTED_FIELD_NAMES );
369
+ gitopsObjectMapper .configure (DeserializationFeature .FAIL_ON_UNKNOWN_PROPERTIES , false );
370
+ gitopsObjectMapper .registerModule (new Jdk8Module ());
371
+ DefaultIndenter defaultIndenter = new DefaultIndenter (" " , DefaultIndenter .SYS_LF );
372
+ DefaultPrettyPrinter printer = new DefaultPrettyPrinter ()
373
+ .withObjectIndenter (defaultIndenter )
374
+ .withArrayIndenter (defaultIndenter );
375
+ gitopsObjectMapper .setDefaultPrettyPrinter (printer );
376
+ return gitopsObjectMapper ;
368
377
}
369
378
370
379
private void initializeLogger (boolean verbose ) {
0 commit comments