From 17e97581ad2a96731b09e6e3798d2e6926b6bd32 Mon Sep 17 00:00:00 2001 From: Travis Ball Date: Sat, 27 Mar 2021 15:59:41 -0400 Subject: [PATCH 01/10] feat: added ability to manage schema registry --- build.gradle | 6 + .../devshawn/kafka/gitops/StateManager.java | 58 ++++++++- .../kafka/gitops/cli/ApplyCommand.java | 3 + .../kafka/gitops/cli/PlanCommand.java | 3 + .../gitops/config/SchemaRegistryConfig.java | 18 +++ .../config/SchemaRegistryConfigLoader.java | 82 +++++++++++++ .../kafka/gitops/domain/plan/DesiredPlan.java | 3 + .../kafka/gitops/domain/plan/SchemaPlan.java | 22 ++++ .../gitops/domain/state/DesiredState.java | 2 + .../gitops/domain/state/DesiredStateFile.java | 2 + .../gitops/domain/state/ReferenceDetails.java | 19 +++ .../gitops/domain/state/SchemaDetails.java | 22 ++++ .../domain/state/settings/Settings.java | 2 + .../state/settings/SettingsDirectory.java | 17 +++ .../state/settings/SettingsRegistry.java | 17 +++ .../domain/state/settings/SettingsSchema.java | 18 +++ .../exception/CompareSchemasException.java | 8 ++ ...MissingMultipleConfigurationException.java | 13 +++ .../SchemaRegistryExecutionException.java | 15 +++ .../kafka/gitops/manager/ApplyManager.java | 23 +++- .../kafka/gitops/manager/PlanManager.java | 50 +++++++- .../kafka/gitops/service/ParserService.java | 10 +- .../gitops/service/SchemaRegistryService.java | 110 ++++++++++++++++++ .../devshawn/kafka/gitops/util/LogUtil.java | 56 +++++++++ .../devshawn/kafka/gitops/util/PlanUtil.java | 7 ++ 25 files changed, 578 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfig.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsDirectory.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsRegistry.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/exception/CompareSchemasException.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/exception/MissingMultipleConfigurationException.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/exception/SchemaRegistryExecutionException.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java diff --git a/build.gradle b/build.gradle index aa2e54de..491015dd 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,9 @@ sourceCompatibility = 1.8 repositories { mavenCentral() + maven { + url "https://packages.confluent.io/maven/" + } } dependencies { @@ -28,6 +31,9 @@ dependencies { compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.10.2" compile 'info.picocli:picocli:4.1.4' + implementation ('io.confluent:kafka-schema-registry-client:6.1.1') + implementation('com.flipkart.zjsonpatch:zjsonpatch:0.4.11') + compile 'org.slf4j:slf4j-api:1.7.30' compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.3' diff --git a/src/main/java/com/devshawn/kafka/gitops/StateManager.java b/src/main/java/com/devshawn/kafka/gitops/StateManager.java index c91c97b8..cf747385 100644 --- a/src/main/java/com/devshawn/kafka/gitops/StateManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/StateManager.java @@ -2,7 +2,6 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; -import com.devshawn.kafka.gitops.config.KafkaGitopsConfig; import com.devshawn.kafka.gitops.config.KafkaGitopsConfigLoader; import com.devshawn.kafka.gitops.config.ManagerConfig; import com.devshawn.kafka.gitops.domain.confluent.ServiceAccount; @@ -61,19 +60,20 @@ public StateManager(ManagerConfig managerConfig, ParserService parserService) { initializeLogger(managerConfig.isVerboseRequested()); this.managerConfig = managerConfig; this.objectMapper = initializeObjectMapper(); - KafkaGitopsConfig config = KafkaGitopsConfigLoader.load(); - this.kafkaService = new KafkaService(config); + this.kafkaService = new KafkaService(KafkaGitopsConfigLoader.load()); + this.schemaRegistryService = new SchemaRegistryService(SchemaRegistryConfigLoader.load()); this.parserService = parserService; this.roleService = new RoleService(); this.confluentCloudService = new ConfluentCloudService(objectMapper); - this.planManager = new PlanManager(managerConfig, kafkaService, objectMapper); - this.applyManager = new ApplyManager(managerConfig, kafkaService); + this.planManager = new PlanManager(managerConfig, kafkaService, schemaRegistryService, objectMapper); + this.applyManager = new ApplyManager(managerConfig, kafkaService, schemaRegistryService); } public DesiredStateFile getAndValidateStateFile() { DesiredStateFile desiredStateFile = parserService.parseStateFile(); validateTopics(desiredStateFile); validateCustomAcls(desiredStateFile); + validateSchemas(desiredStateFile); this.describeAclEnabled = StateUtil.isDescribeTopicAclEnabled(desiredStateFile); return desiredStateFile; } @@ -92,6 +92,7 @@ private DesiredPlan generatePlan() { planManager.planAcls(desiredState, desiredPlan); } planManager.planTopics(desiredState, desiredPlan); + planManager.planSchemas(desiredState, desiredPlan); return desiredPlan.build(); } @@ -107,6 +108,7 @@ public DesiredPlan apply() { if (!managerConfig.isSkipAclsDisabled()) { applyManager.applyAcls(desiredPlan); } + applyManager.applySchemas(desiredPlan); return desiredPlan; } @@ -147,6 +149,7 @@ private DesiredState getDesiredState() { .addAllPrefixedTopicsToIgnore(getPrefixedTopicsToIgnore(desiredStateFile)); generateTopicsState(desiredState, desiredStateFile); + generateSchemasState(desiredState, desiredStateFile); if (isConfluentCloudEnabled(desiredStateFile)) { generateConfluentCloudServiceAcls(desiredState, desiredStateFile); @@ -171,6 +174,10 @@ private void generateTopicsState(DesiredState.Builder desiredState, DesiredState } } + private void generateSchemasState(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { + desiredState.putAllSchemas(desiredStateFile.getSchemas()); + } + private void generateConfluentCloudServiceAcls(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { List serviceAccounts = confluentCloudService.getServiceAccounts(); desiredStateFile.getServices().forEach((name, service) -> { @@ -323,6 +330,47 @@ private void validateTopics(DesiredStateFile desiredStateFile) { } } + private void validateSchemas(DesiredStateFile desiredStateFile) { + if (!desiredStateFile.getSchemas().isEmpty()) { + SchemaRegistryConfig schemaRegistryConfig = SchemaRegistryConfigLoader.load(); + desiredStateFile.getSchemas().forEach((s, schemaDetails) -> { + if (!schemaDetails.getType().equalsIgnoreCase("Avro")) { + throw new ValidationException(String.format("Schema type %s is currently not supported.", schemaDetails.getType())); + } + if (!Files.exists(Paths.get(schemaRegistryConfig.getConfig().get("SCHEMA_DIRECTORY") + "/" + schemaDetails.getFile()))) { + throw new ValidationException(String.format("Schema file %s not found in schema directory at %s", schemaDetails.getFile(), schemaRegistryConfig.getConfig().get("SCHEMA_DIRECTORY"))); + } + if (schemaDetails.getType().equalsIgnoreCase("Avro")) { + AvroSchemaProvider avroSchemaProvider = new AvroSchemaProvider(); + if (schemaDetails.getReferences().isEmpty() && schemaDetails.getType().equalsIgnoreCase("Avro")) { + Optional parsedSchema = avroSchemaProvider.parseSchema(schemaRegistryService.loadSchemaFromDisk(schemaDetails.getFile()), Collections.emptyList()); + if (!parsedSchema.isPresent()) { + throw new ValidationException(String.format("Avro schema %s could not be parsed.", schemaDetails.getFile())); + } + } else { + List schemaReferences = new ArrayList<>(); + schemaDetails.getReferences().forEach(referenceDetails -> { + SchemaReference schemaReference = new SchemaReference(referenceDetails.getName(), referenceDetails.getSubject(), referenceDetails.getVersion()); + schemaReferences.add(schemaReference); + }); + // we need to pass a schema registry client as a config because the underlying code validates against the current state + avroSchemaProvider.configure(Collections.singletonMap(SchemaProvider.SCHEMA_VERSION_FETCHER_CONFIG, schemaRegistryService.createSchemaRegistryClient())); + try { + Optional parsedSchema = avroSchemaProvider.parseSchema(schemaRegistryService.loadSchemaFromDisk(schemaDetails.getFile()), schemaReferences); + if (!parsedSchema.isPresent()) { + throw new ValidationException(String.format("Avro schema %s could not be parsed.", schemaDetails.getFile())); + } + } catch (IllegalStateException ex) { + throw new ValidationException(String.format("Reference validation error: %s", ex.getMessage())); + } catch (RuntimeException ex) { + throw new ValidationException(String.format("Error thrown when attempting to validate schema with reference", ex.getMessage())); + } + } + } + }); + } + } + private boolean isConfluentCloudEnabled(DesiredStateFile desiredStateFile) { if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getCcloud().isPresent()) { return desiredStateFile.getSettings().get().getCcloud().get().isEnabled(); diff --git a/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java b/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java index 0c94bd96..52629287 100644 --- a/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java @@ -8,6 +8,7 @@ import com.devshawn.kafka.gitops.exception.MissingConfigurationException; import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; import com.devshawn.kafka.gitops.exception.ReadPlanInputException; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.ValidationException; import com.devshawn.kafka.gitops.service.ParserService; import com.devshawn.kafka.gitops.util.LogUtil; @@ -45,6 +46,8 @@ public Integer call() { LogUtil.printValidationResult(ex.getMessage(), false); } catch (KafkaExecutionException ex) { LogUtil.printKafkaExecutionError(ex, true); + } catch (SchemaRegistryExecutionException ex) { + LogUtil.printSchemaRegistryExecutionError(ex, true); } return 2; } diff --git a/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java b/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java index f8c4e8f4..caa033ca 100644 --- a/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java @@ -7,6 +7,7 @@ import com.devshawn.kafka.gitops.exception.KafkaExecutionException; import com.devshawn.kafka.gitops.exception.MissingConfigurationException; import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.ValidationException; import com.devshawn.kafka.gitops.exception.WritePlanOutputException; import com.devshawn.kafka.gitops.service.ParserService; @@ -49,6 +50,8 @@ public Integer call() { LogUtil.printKafkaExecutionError(ex); } catch (WritePlanOutputException ex) { LogUtil.printPlanOutputError(ex); + } catch (SchemaRegistryExecutionException ex) { + LogUtil.printSchemaRegistryExecutionError(ex); } return 2; } diff --git a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfig.java b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfig.java new file mode 100644 index 00000000..82e1db6c --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfig.java @@ -0,0 +1,18 @@ +package com.devshawn.kafka.gitops.config; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.Map; + +@FreeBuilder +@JsonDeserialize(builder = SchemaRegistryConfig.Builder.class) +public interface SchemaRegistryConfig { + + Map getConfig(); + + class Builder extends SchemaRegistryConfig_Builder { + + } + +} diff --git a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java new file mode 100644 index 00000000..e2497dc7 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java @@ -0,0 +1,82 @@ +package com.devshawn.kafka.gitops.config; + +import com.devshawn.kafka.gitops.exception.MissingConfigurationException; +import com.devshawn.kafka.gitops.exception.MissingMultipleConfigurationException; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +public class SchemaRegistryConfigLoader { + + private static org.slf4j.Logger log = LoggerFactory.getLogger(SchemaRegistryConfigLoader.class); + + public static SchemaRegistryConfig load() { + SchemaRegistryConfig.Builder builder = new SchemaRegistryConfig.Builder(); + setConfig(builder); + return builder.build(); + } + + private static void setConfig(SchemaRegistryConfig.Builder builder) { + Map config = new HashMap<>(); + AtomicReference username = new AtomicReference<>(); + AtomicReference password = new AtomicReference<>(); + + Map environment = System.getenv(); + + environment.forEach((key, value) -> { + if (key.equals("SCHEMA_REGISTRY_SASL_JAAS_USERNAME")) { + username.set(value); + } else if (key.equals("SCHEMA_REGISTRY_SASL_JAAS_PASSWORD")) { + password.set(value); + } else if (key.equals("SCHEMA_REGISTRY_URL")) { + config.put("SCHEMA_REGISTRY_URL", value); + } else if (key.equals("SCHEMA_DIRECTORY")) { + config.put("SCHEMA_DIRECTORY", value); + } + }); + + handleDefaultConfig(config); + handleAuthentication(username, password, config); + + log.info("Schema Registry Config: {}", config); + + builder.putAllConfig(config); + } + + private static void handleDefaultConfig(Map config) { + final String DEFAULT_URL = "http://localhost:8081"; + final String CURRENT_WORKING_DIR = System.getProperty("user.dir"); + if (!config.containsKey("SCHEMA_REGISTRY_URL")) { + log.info("SCHEMA_REGISTRY_URL not set. Using default value of {}", DEFAULT_URL); + config.put("SCHEMA_REGISTRY_URL", DEFAULT_URL); + } + if (!config.containsKey("SCHEMA_DIRECTORY")) { + log.info("SCHEMA_DIRECTORY not set. Defaulting to current working directory: {}", CURRENT_WORKING_DIR); + config.put("SCHEMA_DIRECTORY", CURRENT_WORKING_DIR); + } + } + + private static void handleAuthentication(AtomicReference username, AtomicReference password, Map config) { + if (username.get() != null && password.get() != null) { + String loginModule = "org.apache.kafka.common.security.plain.PlainLoginModule"; + String value = String.format("%s required username=\"%s\" password=\"%s\";", + loginModule, escape(username.get()), escape(password.get())); + config.put("SCHEMA_REGISTRY_SASL_CONFIG", value); + } else if (username.get() != null) { + throw new MissingConfigurationException("SCHEMA_REGISTRY_SASL_JAAS_USERNAME"); + } else if (password.get() != null) { + throw new MissingConfigurationException("SCHEMA_REGISTRY_SASL_JAAS_PASSWORD"); + } else if (username.get() == null & password.get() == null) { + throw new MissingMultipleConfigurationException("SCHEMA_REGISTRY_SASL_JAAS_PASSWORD", "SCHEMA_REGISTRY_SASL_JAAS_USERNAME"); + } + } + + private static String escape(String value) { + if (value != null) { + return value.replace("\"", "\\\""); + } + return null; + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java b/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java index 156c6284..7eca2561 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java @@ -12,12 +12,15 @@ public interface DesiredPlan { List getTopicPlans(); + List getSchemaPlans(); + List getAclPlans(); default DesiredPlan toChangesOnlyPlan() { DesiredPlan.Builder builder = new DesiredPlan.Builder(); getTopicPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).map(TopicPlan::toChangesOnlyPlan).forEach(builder::addTopicPlans); getAclPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).forEach(builder::addAclPlans); + getSchemaPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).forEach(builder::addSchemaPlans); return builder.build(); } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java b/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java new file mode 100644 index 00000000..29dc53b0 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java @@ -0,0 +1,22 @@ +package com.devshawn.kafka.gitops.domain.plan; + +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; +import com.devshawn.kafka.gitops.enums.PlanAction; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SchemaPlan.Builder.class) +public interface SchemaPlan { + + String getName(); + + PlanAction getAction(); + + Optional getSchemaDetails(); + + class Builder extends SchemaPlan_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java index d46d6d78..feb34a92 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java @@ -14,6 +14,8 @@ public interface DesiredState { Map getAcls(); + Map getSchemas(); + List getPrefixedTopicsToIgnore(); class Builder extends DesiredState_Builder { diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java index a6862fb9..35689da5 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java @@ -18,6 +18,8 @@ public interface DesiredStateFile { Map getTopics(); + Map getSchemas(); + Map getUsers(); Map> getCustomServiceAcls(); diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java new file mode 100644 index 00000000..d8dbd41e --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java @@ -0,0 +1,19 @@ +package com.devshawn.kafka.gitops.domain.state; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + + +@FreeBuilder +@JsonDeserialize(builder = ReferenceDetails.Builder.class) +public interface ReferenceDetails { + + String getName(); + + String getSubject(); + + Integer getVersion(); + + class Builder extends ReferenceDetails_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java new file mode 100644 index 00000000..47f5a9d7 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java @@ -0,0 +1,22 @@ +package com.devshawn.kafka.gitops.domain.state; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.List; + +@FreeBuilder +@JsonDeserialize(builder = SchemaDetails.Builder.class) +public interface SchemaDetails { + + String getType(); + + String getFile(); + + List getSubjects(); + + List getReferences(); + + class Builder extends SchemaDetails_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java index 0753769a..7f1b60e0 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java @@ -17,6 +17,8 @@ public interface Settings { Optional getFiles(); + Optional getSchema(); + class Builder extends Settings_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsDirectory.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsDirectory.java new file mode 100644 index 00000000..d065aeec --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsDirectory.java @@ -0,0 +1,17 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.nio.file.Path; +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SettingsDirectory.Builder.class) +public interface SettingsDirectory { + + Optional getPath(); + + class Builder extends SettingsDirectory_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsRegistry.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsRegistry.java new file mode 100644 index 00000000..404379e8 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsRegistry.java @@ -0,0 +1,17 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.net.URL; +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SettingsRegistry.Builder.class) +public interface SettingsRegistry { + + Optional getUrl(); + + class Builder extends SettingsRegistry_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java new file mode 100644 index 00000000..7e0eab3e --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java @@ -0,0 +1,18 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.Optional; + +@FreeBuilder +@JsonDeserialize(builder = SettingsSchema.Builder.class) +public interface SettingsSchema { + + Optional getRegistry(); + + Optional getDirectory(); + + class Builder extends SettingsSchema_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/exception/CompareSchemasException.java b/src/main/java/com/devshawn/kafka/gitops/exception/CompareSchemasException.java new file mode 100644 index 00000000..c0766d6d --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/exception/CompareSchemasException.java @@ -0,0 +1,8 @@ +package com.devshawn.kafka.gitops.exception; + +public class CompareSchemasException extends RuntimeException { + + public CompareSchemasException(String exMessage) { + super(String.format("Error comparing schemas: %s", exMessage)); + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/exception/MissingMultipleConfigurationException.java b/src/main/java/com/devshawn/kafka/gitops/exception/MissingMultipleConfigurationException.java new file mode 100644 index 00000000..9b086cc3 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/exception/MissingMultipleConfigurationException.java @@ -0,0 +1,13 @@ +package com.devshawn.kafka.gitops.exception; + +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + +public class MissingMultipleConfigurationException extends RuntimeException { + + public MissingMultipleConfigurationException(String... environmentVariableNames) { + super(String.format("Missing required configuration(s): %s", + Arrays.stream(environmentVariableNames).map(Objects::toString).collect(Collectors.joining(", ")))); + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/exception/SchemaRegistryExecutionException.java b/src/main/java/com/devshawn/kafka/gitops/exception/SchemaRegistryExecutionException.java new file mode 100644 index 00000000..cba74ca4 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/exception/SchemaRegistryExecutionException.java @@ -0,0 +1,15 @@ +package com.devshawn.kafka.gitops.exception; + +public class SchemaRegistryExecutionException extends RuntimeException { + + private final String exceptionMessage; + + public SchemaRegistryExecutionException(String message, String exceptionMessage) { + super(message); + this.exceptionMessage = exceptionMessage; + } + + public String getExceptionMessage() { + return exceptionMessage; + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java index c9704ded..12a28665 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java @@ -7,6 +7,7 @@ import com.devshawn.kafka.gitops.domain.plan.TopicPlan; import com.devshawn.kafka.gitops.enums.PlanAction; import com.devshawn.kafka.gitops.service.KafkaService; +import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.LogUtil; import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.ConfigEntry; @@ -19,10 +20,12 @@ public class ApplyManager { private final ManagerConfig managerConfig; private final KafkaService kafkaService; + private final SchemaRegistryService schemaRegistryService; - public ApplyManager(ManagerConfig managerConfig, KafkaService kafkaService) { + public ApplyManager(ManagerConfig managerConfig, KafkaService kafkaService, SchemaRegistryService schemaRegistryService) { this.managerConfig = managerConfig; this.kafkaService = kafkaService; + this.schemaRegistryService = schemaRegistryService; } public void applyTopics(DesiredPlan desiredPlan) { @@ -89,4 +92,22 @@ public void applyAcls(DesiredPlan desiredPlan) { } }); } + + public void applySchemas(DesiredPlan desiredPlan) { + desiredPlan.getSchemaPlans().forEach(schemaPlan -> { + if (schemaPlan.getAction() == PlanAction.ADD) { + LogUtil.printSchemaPreApply(schemaPlan); + schemaRegistryService.register(schemaPlan); + LogUtil.printPostApply(); + } else if (schemaPlan.getAction() == PlanAction.UPDATE) { + LogUtil.printSchemaPreApply(schemaPlan); + schemaRegistryService.register(schemaPlan); + LogUtil.printPostApply(); + } else if (schemaPlan.getAction() == PlanAction.REMOVE && !managerConfig.isDeleteDisabled()) { + LogUtil.printSchemaPreApply(schemaPlan); + schemaRegistryService.deleteSubject(schemaPlan.getName(), true); + LogUtil.printPostApply(); + } + }); + } } diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java index 4cd52b85..0dbf1543 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java @@ -4,6 +4,7 @@ import com.devshawn.kafka.gitops.domain.plan.AclPlan; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; import com.devshawn.kafka.gitops.domain.plan.PlanOverview; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; import com.devshawn.kafka.gitops.domain.plan.TopicConfigPlan; import com.devshawn.kafka.gitops.domain.plan.TopicDetailsPlan; import com.devshawn.kafka.gitops.domain.plan.TopicPlan; @@ -16,8 +17,10 @@ import com.devshawn.kafka.gitops.exception.ValidationException; import com.devshawn.kafka.gitops.exception.WritePlanOutputException; import com.devshawn.kafka.gitops.service.KafkaService; +import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.PlanUtil; import com.fasterxml.jackson.databind.ObjectMapper; +import io.confluent.kafka.schemaregistry.client.SchemaMetadata; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; @@ -40,11 +43,13 @@ public class PlanManager { private final ManagerConfig managerConfig; private final KafkaService kafkaService; + private final SchemaRegistryService schemaRegistryService; private final ObjectMapper objectMapper; - public PlanManager(ManagerConfig managerConfig, KafkaService kafkaService, ObjectMapper objectMapper) { + public PlanManager(ManagerConfig managerConfig, KafkaService kafkaService, SchemaRegistryService schemaRegistryService, ObjectMapper objectMapper) { this.managerConfig = managerConfig; this.kafkaService = kafkaService; + this.schemaRegistryService = schemaRegistryService; this.objectMapper = objectMapper; } @@ -220,6 +225,49 @@ public void planAcls(DesiredState desiredState, DesiredPlan.Builder desiredPlan) }); } + public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPlan) { + // TODO: Parallelize getting schema metadata? + Map currentSubjectSchemasMap = new HashMap<>(); + schemaRegistryService.getAllSubjects().forEach(subject -> { + SchemaMetadata schemaMetadata = schemaRegistryService.getLatestSchemaMetadata(subject); + currentSubjectSchemasMap.put(subject, schemaMetadata); + }); + + desiredState.getSchemas().forEach((subject, schemaDetails) -> { + SchemaPlan.Builder schemaPlan = new SchemaPlan.Builder() + .setName(subject) + .setSchemaDetails(schemaDetails); + + if (!currentSubjectSchemasMap.containsKey(subject)) { + log.info("[PLAN] Schema Subject {} does not exist; it will be created.", subject); + schemaPlan.setAction(PlanAction.ADD); + } else { + String diff = schemaRegistryService.compareSchemasAndReturnDiff(schemaRegistryService.loadSchemaFromDisk(schemaDetails.getFile()), currentSubjectSchemasMap.get(subject).getSchema()); + if (diff == null) { + log.info("[PLAN] Schema Subject {} exists and has not changed; it will not be created.", subject); + schemaPlan.setAction(PlanAction.NO_CHANGE); + } else { + log.info("[PLAN] Schema Subject {} exists and has changed; it will be updated.", subject); + schemaPlan.setAction(PlanAction.UPDATE); + // TODO: Set diff string for logging? + } + } + + desiredPlan.addSchemaPlans(schemaPlan.build()); + }); + + currentSubjectSchemasMap.forEach((subject, schemaMetadata) -> { + if (!managerConfig.isDeleteDisabled() && desiredState.getSchemas().getOrDefault(subject, null) == null) { + SchemaPlan schemaPlan = new SchemaPlan.Builder() + .setName(subject) + .setAction(PlanAction.REMOVE) + .build(); + + desiredPlan.addSchemaPlans(schemaPlan); + } + }); + } + public void validatePlanHasChanges(DesiredPlan desiredPlan, boolean deleteDisabled, boolean skipAclsDisabled) { PlanOverview planOverview = PlanUtil.getOverview(desiredPlan, deleteDisabled, skipAclsDisabled); if (planOverview.getAdd() == 0 && planOverview.getUpdate() == 0 && planOverview.getRemove() == 0) { diff --git a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java index c027f4af..80b9c682 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java @@ -81,7 +81,15 @@ public DesiredStateFile parseStateFile(File stateFile) { throw new ValidationException(String.format("Value '%s' is not a valid format for: [%s] in state file definition: %s", value, propertyName, joinedFields)); } catch (JsonMappingException ex) { List fields = getYamlFields(ex); - String message = ex.getCause() != null ? ex.getCause().getMessage().split("\n")[0] : ex.getMessage().split("\n")[0]; + String message = null; + if(ex.getCause() != null && ex.getCause().getMessage() != null) { + message = ex.getCause().getMessage().split("\n")[0]; + } + if(message == null && ex.getMessage() != null) { + message = ex.getMessage().split("\n")[0]; + } else { + message = "Unknown error"; + } String joinedFields = String.join(" -> ", fields); throw new ValidationException(String.format("%s in state file definition: %s", message, joinedFields)); } catch (FileNotFoundException ex) { diff --git a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java new file mode 100644 index 00000000..1793d5d1 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java @@ -0,0 +1,110 @@ +package com.devshawn.kafka.gitops.service; + + +import com.devshawn.kafka.gitops.config.SchemaRegistryConfig; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.zjsonpatch.JsonDiff; +import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaMetadata; +import io.confluent.kafka.schemaregistry.client.rest.RestService; +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; +import io.confluent.kafka.schemaregistry.client.security.basicauth.SaslBasicAuthCredentialProvider; +import org.apache.kafka.common.config.SaslConfigs; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; + +public class SchemaRegistryService { + + private final SchemaRegistryConfig config; + + public SchemaRegistryService(SchemaRegistryConfig config) { + this.config = config; + } + + public List getAllSubjects() { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + try { + return new ArrayList<>(cachedSchemaRegistryClient.getAllSubjects()); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to get all schema registry subjects", ex.getMessage()); + } + } + + public void deleteSubject(String subject, boolean isPermanent) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + try { + // must always soft-delete + cachedSchemaRegistryClient.deleteSubject(subject); + if (isPermanent) { + cachedSchemaRegistryClient.deleteSubject(subject, true); + } + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); + } + } + + public int register(SchemaPlan schemaPlan) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + AvroSchemaProvider avroSchemaProvider = new AvroSchemaProvider(); + Optional parsedSchema = avroSchemaProvider.parseSchema(loadSchemaFromDisk(schemaPlan.getSchemaDetails().get().getFile()), Collections.emptyList()); + try { + return cachedSchemaRegistryClient.register(schemaPlan.getName(), parsedSchema.get()); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to register subject with schema registry", ex.getMessage()); + } + } + + public SchemaMetadata getLatestSchemaMetadata(String subject) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + try { + return cachedSchemaRegistryClient.getLatestSchemaMetadata(subject); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); + } + } + + public String compareSchemasAndReturnDiff(String schemaStringOne, String schemaStringTwo) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + JsonNode schemaOne = objectMapper.readTree(schemaStringOne); + JsonNode schemaTwo = objectMapper.readTree(schemaStringTwo); + JsonNode diff = JsonDiff.asJson(schemaOne, schemaTwo); + if (diff.isEmpty()) { + return null; + } + return diff.toString(); + } catch (JsonProcessingException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to compare the schemas", ex.getMessage()); + } + } + + public String loadSchemaFromDisk(String fileName) { + final String SCHEMA_DIRECTORY = config.getConfig().get("SCHEMA_DIRECTORY").toString(); + try { + return new String(Files.readAllBytes(Paths.get(SCHEMA_DIRECTORY + "/" + fileName)), StandardCharsets.UTF_8); + } catch (IOException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to load a schema from schema directory", ex.getMessage()); + } + } + + public CachedSchemaRegistryClient createSchemaRegistryClient() { + RestService restService = new RestService(config.getConfig().get("SCHEMA_REGISTRY_URL").toString()); + SaslBasicAuthCredentialProvider saslBasicAuthCredentialProvider = new SaslBasicAuthCredentialProvider(); + Map clientConfig = new HashMap<>(); + clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, config.getConfig().get("SCHEMA_REGISTRY_SASL_CONFIG").toString()); + saslBasicAuthCredentialProvider.configure(clientConfig); + restService.setBasicAuthCredentialProvider(saslBasicAuthCredentialProvider); + return new CachedSchemaRegistryClient(restService, 10); + } + +} diff --git a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java index 49919693..352abfc8 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java @@ -3,12 +3,14 @@ import com.devshawn.kafka.gitops.domain.plan.AclPlan; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; import com.devshawn.kafka.gitops.domain.plan.PlanOverview; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; import com.devshawn.kafka.gitops.domain.plan.TopicConfigPlan; import com.devshawn.kafka.gitops.domain.plan.TopicDetailsPlan; import com.devshawn.kafka.gitops.domain.plan.TopicPlan; import com.devshawn.kafka.gitops.domain.state.AclDetails; import com.devshawn.kafka.gitops.enums.PlanAction; import com.devshawn.kafka.gitops.exception.KafkaExecutionException; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.WritePlanOutputException; import picocli.CommandLine; @@ -25,6 +27,9 @@ public static void printPlan(DesiredPlan desiredPlan, boolean deleteDisabled, bo printAclOverview(desiredPlan, deleteDisabled); desiredPlan.getAclPlans().forEach(LogUtil::printAclPlan); + printSchemaOverview(desiredPlan, deleteDisabled); + desiredPlan.getSchemaPlans().forEach(LogUtil::printSchemaPlan); + printOverview(desiredPlan, deleteDisabled, skipAclsDisabled); } @@ -145,6 +150,33 @@ private static void printAclPlan(AclPlan aclPlan) { } } + private static void printSchemaPlan(SchemaPlan schemaPlan) { + switch (schemaPlan.getAction()) { + case ADD: + System.out.println(green(String.format("+ [SCHEMA] %s", schemaPlan.getName()))); + System.out.println(green(String.format("\t + type: %s", schemaPlan.getSchemaDetails().get().getType()))); + System.out.println(green(String.format("\t + file: %s", schemaPlan.getSchemaDetails().get().getFile()))); + if (!schemaPlan.getSchemaDetails().get().getReferences().isEmpty()) { + schemaPlan.getSchemaDetails().get().getReferences().forEach(referenceDetail -> { + System.out.println(green(String.format("\t + reference:"))); + System.out.println(green(String.format("\t\t + name: %s", referenceDetail.getName()))); + System.out.println(green(String.format("\t\t + subject: %s", referenceDetail.getSubject()))); + System.out.println(green(String.format("\t\t + version: %s", referenceDetail.getVersion()))); + }); + } + System.out.println("\n"); + break; + case UPDATE: + System.out.println(yellow(String.format("~ [SCHEMA] %s", schemaPlan.getName()))); + System.out.println("\n"); + break; + case REMOVE: + System.out.println(red(String.format("- [SCHEMA] %s", schemaPlan.getName()))); + System.out.println("\n"); + break; + } + } + /* * Apply */ @@ -154,6 +186,11 @@ public static void printTopicPreApply(TopicPlan topicPlan) { printTopicPlan(topicPlan); } + public static void printSchemaPreApply(SchemaPlan schemaPlan) { + System.out.println(String.format("Applying: [%s]\n", toAction(schemaPlan.getAction()))); + printSchemaPlan(schemaPlan); + } + public static void printAclPreApply(AclPlan aclPlan) { System.out.println(String.format("Applying: [%s]\n", toAction(aclPlan.getAction()))); printAclPlan(aclPlan); @@ -185,6 +222,12 @@ private static void printAclOverview(DesiredPlan desiredPlan, boolean deleteDisa toUpdate(aclPlanOverview.getUpdate()), toDelete(aclPlanOverview.getRemove()))); } + private static void printSchemaOverview(DesiredPlan desiredPlan, boolean deleteDisabled) { + PlanOverview schemaPlanOverview = PlanUtil.getSchemaPlanOverview(desiredPlan, deleteDisabled); + System.out.println(String.format("Schemas: %s, %s, %s.\n", toCreate(schemaPlanOverview.getAdd()), + toUpdate(schemaPlanOverview.getUpdate()), toDelete(schemaPlanOverview.getRemove()))); + } + private static void printLegend(PlanOverview planOverview) { System.out.println("An execution plan has been generated and is shown below."); System.out.println("Resource actions are indicated with the following symbols:"); @@ -247,6 +290,19 @@ public static void printKafkaExecutionError(KafkaExecutionException ex, boolean } } + public static void printSchemaRegistryExecutionError(SchemaRegistryExecutionException ex) { + printSchemaRegistryExecutionError(ex, false); + } + + public static void printSchemaRegistryExecutionError(SchemaRegistryExecutionException ex, boolean apply) { + System.out.println(String.format("[%s] %s:\n%s\n", red("ERROR"), ex.getMessage(), ex.getExceptionMessage())); + if (apply) { + printApplyErrorMessage(); + } else { + printPlanErrorMessage(); + } + } + public static void printPlanOutputError(WritePlanOutputException ex) { System.out.println(String.format("[%s] %s", red("ERROR"), ex.getMessage())); } diff --git a/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java index ff86ca95..c5af87b3 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java @@ -15,6 +15,7 @@ public static PlanOverview getOverview(DesiredPlan desiredPlan, boolean deleteDi if(!skipAclsDisabled) { desiredPlan.getAclPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); } + desiredPlan.getSchemaPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); return buildPlanOverview(map); } @@ -30,6 +31,12 @@ public static PlanOverview getAclPlanOverview(DesiredPlan desiredPlan, boolean d return buildPlanOverview(map); } + public static PlanOverview getSchemaPlanOverview(DesiredPlan desiredPlan, boolean deleteDisabled) { + EnumMap map = getPlanActionMap(); + desiredPlan.getSchemaPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); + return buildPlanOverview(map); + } + private static void addToMap(EnumMap map, PlanAction planAction, boolean deleteDisabled) { if (!(deleteDisabled && planAction == PlanAction.REMOVE)) { map.put(planAction, map.get(planAction) + 1); From c4aa228dfd795748599a52d73de2c35e112dfb7a Mon Sep 17 00:00:00 2001 From: Jerome Revillard Date: Thu, 5 Aug 2021 09:34:03 +0200 Subject: [PATCH 02/10] Bump Groovy to 2.5 to easy Eclipse IDE integration --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 491015dd..b6d49522 100644 --- a/build.gradle +++ b/build.gradle @@ -41,8 +41,8 @@ dependencies { processor 'org.inferred:freebuilder:2.5.0' testCompile group: 'junit', name: 'junit', version: '4.12' - testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.4.4' - testCompile group: 'org.spockframework', name: 'spock-core', version: '1.0-groovy-2.4' + testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.5.14' + testCompile group: 'org.spockframework', name: 'spock-core', version: '1.2-groovy-2.5' testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2' testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0' testCompile group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.0' From 510452480ab8f8aa5c7294dde40f287e58419b10 Mon Sep 17 00:00:00 2001 From: Jerome Revillard Date: Mon, 9 Aug 2021 11:24:34 +0200 Subject: [PATCH 03/10] Bump to Gradle 5.6.4 to fix Eclipse IDE integration --- build.gradle | 6 +++--- gradle/wrapper/gradle-wrapper.jar | Bin 56172 -> 55741 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 8 ++++---- gradlew.bat | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index b6d49522..23d95f37 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ plugins { id 'application' id 'idea' id 'jacoco' - id 'org.inferred.processors' version '1.2.10' + id 'org.inferred.processors' version '2.1.0' id "net.ltgt.apt" version "0.21" id 'com.github.johnrengelman.shadow' version '4.0.4' } @@ -38,7 +38,7 @@ dependencies { compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.3' - processor 'org.inferred:freebuilder:2.5.0' + processor 'org.inferred:freebuilder:2.7.0' testCompile group: 'junit', name: 'junit', version: '4.12' testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.5.14' @@ -52,7 +52,7 @@ jacocoTestReport { reports { xml.enabled = true html.enabled = true - xml.destination "${buildDir}/reports/jacoco/report.xml" + xml.destination file("${buildDir}/reports/jacoco/report.xml") } afterEvaluate { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 28861d273a5d270fd8f65dd74570c17c9c507736..457aad0d98108420a977756b7145c93c8910b076 100644 GIT binary patch delta 46490 zcmZ6xV{j&3@TQ%~#I|iac`~tW+qRuN6WjL0wr$(CZEJt;f2+3M-Kuk{&d07gU3Yh1 zeP1^J?rQiQ2Y}{jxSQhz0Rd5o7f^x-XuNm&1rmo7-{=NkzGk-i8yp1W92x|KKJg9< zBT@DjK2RN19ql_N-NXP7+*lwmA`pPW5Z<(`R@*4mMf8jQmyO5JHw>&NBB%a!l*;f;9@Kez)7KULU% zK)^4VaS!QXQSh$T!d;eXsL|mzH7m<>?uYk!$pnkx>5Jq1Ypne)hNMl^g8sP4RXJu|m&bsyg%|uTonyK8 z0HAFLda9n8%fNVh`+<{r7O@Vpc8GC!(Ioy?P7a4_McQX(!{lyafsZ{WBIkhv*(7>+i&;zMcU-3Dojr3gYMv=t;c?y;ynqh zR4Qxv!P*kyW=fR2p*J{p^(#w2!F8SD{#7p_sC8OBl-dPkcnp9sH6v>!r?; zQ#8sB71F?vWQP?7+k_i?BF&A{6j1jl15pOn$4Y%>#xevw3>IRkSECvRaeeo0+(2rm z3&3G=u(Y+a=O&Pyg5fazUcoxYZzPCS*9 z;skz$LhA(hsNKI;dJu)TR z+uv;p7=QHG-pV5rv_(}GJ6oCI#I+%$a775LtaKb1icLt`iqcJlI4dI0HYcBk!EWr5 z<7Fk?&m!VRa@=^f$^Owc1BIpN1vcWAAx^~(o2N0)5_tr6%oaMCb4Gx;Fwu zAtNG{Cr|5U0AZPqO{bXT;!iV`&A-gYX@q9ZC{L&0#b2rnf$(?f@g=BvtV*#i zr{Z`7XYq2-309|(1f_es#v=QB`UH}fB_}^TeR(>K{JU28p)3Es&XUz;7o^Le6WBm` zrE2Bpzv22YTr3I;@^DeXG%RH8kyx+h`gjyNkiA0Eq5xRjEB>z zARtgs|B0s{h=t;}Z~lL;`Ts8Ie`-mP*gFB0xD5dd+C-Ch0YM1FK88o^e$l%bLG8s6 zh8SidA~W}PPT$1TLK)WHz_JzXrW?{lqTo)nynro;sqbFTA|{@!RMz~e!z-GQ%r2gDu zS{x2aqLqXK%D}>`+&cS&9Ba1Ld!)=Pq%U%QIDappX{Ry_$1K`bXm+~y%%7LdwsifT zo*ys=!dQG6gJu6nOHAb&P`jtDFdO4M_#h`aTWYi62uLu3@ATuGF>b%6dDo5x!bmN<^P8 z_J~k2>9!SgKN_vlQegXjlLQ$4P^N5N0A@X^c*07`G)xj!@wt#aQWvSJEfMB;n&I4x z`t}1NPgpQQNT3wQH4EJDkVLw54Oh!{M49+d4gzMh?;95Eddgurax*cG zskk#_Qi7GhDLd&%-4JM2OmgV54!8+FQ;>XBtQG0E2I3G(;lDo;Q&}j%He&MT3A6qM()B@}E*#8b>0XvVur3fT@Wy#Kre-IQ3tC$`lT-d}SieE? zw6{l)TZiOjvii}du^K8Xr#3=~&WD6htcPLV52P~tRtJ!iiSR6OHiKLf{<$|+$xB3u zlbAy2v|$zfcj_v&9ZcM7T5mAG=?6>4McVb8-Qz85>pZQk;)SP9NT4-DisaX?|C2!f zlRv2Evh0NjARzYmARxs5lRvmH7{KYHd!s;l7o4a*3=R&IQ~{_wQ4-kUXQJrB;H4NI zZdfYWeNOtS%%Dcg12T%3>_NG#ZB7nayo6O(tO;BK7myvbhoC?8 z78k;gH3-R%HfZH`gP12~M{(+gCSyl=>L#=n8)&eL2>+=FC4GB@)c<;b#23M@H~}or zs7ZUZf+Fn8Ejylp{|tWrHieSkwI=>V|JGs$Dofkx1MXUgKk>h}d->B|N02ynbFKb; zvw>3RA@1egMRR?^ezONj1788CKh<_ZPf65Nu7?*sc*?+1seiWx#R3Gq{ zI`d54tHW=Cnf_uc8$@6!q4&%A%q3a*kqe-dDRvP}k%>X^8f7Tq_CKJo6ip*Z%PwWk z8;q0u#)P+o8#98=tPd&~R}xOauHQ^TFRo-GGa4^P3NdM-wwBSX08ZMGhSWE+{ab?>y@f{+GWkYOGGm@f-yQ+QdL412Xmh#> z^@Ji@fQsP^LjZdQ)`Z9OCe^H2jh&Ln6Y~xMpYv$Wq_uI(;Tnvc#dvn6(`QVde-2M- z&_*2~-h~ZvfB;)DpSgHk5}S)uZe70cz7uEsg$3j}i_nc?i39+^@OUot8IfL-gh{ON6IUHn$v(~JDVrSpTVPUCa^0A+b5S__>G)ocWV~f3 z#%wrCNQ8a{Yv$HU+pC-8W{Ry%AvTV!map{J3UC0Mg1Pi3>yDC@FZy~FRi|3Y;iG=v zO5VzCF3wC?3(k_ga4ODCn2XAOoxl~ME689APuo5NXf2OJW;j}PD4XfNb~bpc1GFnY zt$vfD8VL@GG;Y_yZlYvAN8yuOvozmW<}EYK{sjQ2hE+FR&eDoU4wT3_C*HSbbbQsej65emi#AmTm19YO_g|3*30`Ts zsU=+kXHQovFi(@G0)}MH_hjauv-|SN?J6(hoSK@BlyHtGB?-KypGsw0FFEn60*YkO z<)O;jh$-3g|F$szMhEuy*NXOaW@S$lnwDi#<*x5Tun+UnGVen0lZ~RTgbZ3LoawTL zHdG&K!7&pt2m;KBt8fUhVQ+c|*mhcP$*d| zHPbyy9^*Y4y+Y`uUEB5J6c@=B@(xo(#tLw>T9cZ~29usgET`Utz`GmeL`?%pQBI z)&AW%-M({uY`^K-e@^GVOU0rs{@92Bw;EzFo0O9{uHYWZ!=_1u7u^fMO=6ssU2-`L zT!Jv;Q#q3r*{t7S;aSlQB0s>OUZDOo9ECCX4JcZj}#W1pnSiA#>f{I zP_M)mvnD1om|}%H*+sk}^D?;rB}QlkX|tr$pMAzlH}}C+_||eqc#m46$jwzTtca}n zGeg;F`Ns#A@;4of+=d+%uKu9G*Qs;ZS{rD0F>lo|i(wu2I$xWk5)0lY7 zTpQ6hxDkVG=!~}UEF)FXZr!MK-6v~BK$fB0&x>5s!^RXG;<&I@mG+Iy1BD;16)8m6um&s2YDdKOeywRx5FFJ-~A0j6ul#i^WtJoE`RLV%z zrv5ZkdcZ1I{5r{lqg~Jh3U4Xp4VUT20~8imeyXVyn;I7A6186VS*tl?*dJ**6{3i$LYqK0o+W zPi5I<;*Y9|GCM|W8H;T)aooluigYuyhPa9>-+pcXNvd}ExQ*Tin$MfW<0!W?q$_9Q zbN(}_{&Pa^1MyFAgBxd%! zazO;kS#cZseT>c^Fttz4gio_OuGlxZZ|yZViXPNPvC0OLthIfC;jxJwkKDzVG?S%^uQnM@tal zg=1+}Mu638Y-u*MLg%t_%d%^A(5rNGRl9ln7E&0Q)iHEeHJ?+<=nTPYiV%*}&lvc= z4%68I+e6^*Zig*5Kj-h?O`9+;ubw6@9$V}@@w+$_Kl~}@=08Ep2Qup|_=+jBuUxaB z^~5!NwNio^=xTt-^n#6|ljo*QnvHzi4+LE$^pVdJQ?6<88FXCh)dlISEqsxgT$rkQ zfJHBpIq8|Rf8Ovl3nn!OxbKhmem3oBESCB?h9r5&s zpnc#e?_4+KsrO$oPPTiY_lyOs9C|crn?82LC2?+9{P1pyg9x6J0m(Z~<6UW3fE?d&S}5up0frM$JRo z+K`eh%5VNzha^CpprYiMmCCNcEOuf6J)AY}c(ilWYIB9ooTuFCQGRozX?g$K6{(2*nIB z9_G24J5R7(Us`*k`=9h@(;p@e#Yoq>fb0x`*un4k!gC)iv{L%mV{~Ld59odZ&E}>i zvxf5V7>cPvrIlsp;|SD{*7D&-I>K8Tu(}$;h`A%N>QQqG11-ryzDw8`x|I)HmvBo9 zVpA}ZR|QDA51g5(C0XE|j!35JxFdN(wFbN!Rx!8rzdRpJ9RJm@`zSR??&lEkJE`i7zPGI;><-H{i zTS+!Fvuo}t#BHCQYs}c2o-AW_)aSU)33!3(`i^`he};fUN?lX-@U61|UPP`}(*e@- zX7*T<6g6AHRas5+tBe!NZaZPlR^e_E1so$#y;8LGdXqoFUBgL_Ca3l|8B2xt4s1TO zY7QXsH_v@KZ@t!*R?~;Vn{_&M%v`qI+pw4?p6K7O@LvVTzln?ILZKZ~rASPj72l`0 zQZ%a-_A9%m=t0L^n3c|fL_4AtDeiwG)$fe-rfcHDVI~vz5|e5HK0PVVzsPN01)H<| zYpeL!_mt03;Sd2C>wQr_0(1nsB@`IuX@mQHP~Ku>Sdp3ZbsomG$*<~-W?dmwLlx|bxuEx(A;+L^%>QqwO$1KjBtBy8LKg&v zZdOBpfcPYCVRHl3H=J?Rv48M2=hrJ-eJsn>rex({B;;(0_7X@kkw%)7-x6Bu4wlKu zbfqbuhbwgJw1${p(1c)v9jyIBKnGxb4bt*hRU~iW_Q2EjutR$Oj&$Gl!@;HPtR>^^ zpcE5)T>ja4yWY9^aQiy>`F?f>iP5`59u_g z)ku%sx%mu8cUbH@ax7GS=YhmOqbnO zGzw8G<|r2_%oHa0vQofhm0q#dxZmk8UCUF?i}kpZKAE@0V!ff*yv#We>j{(IYD~6u zTga}DRl0eS-rKB*q-%4uDPOljOZ}+SbRy44QLO@)SZvH75gRi+^_Pe#3HqRRGix6g zN}heE-dS{odYMO_%g=7L(bm+FsDt>t;X&PGPJR5aDdMP#X^^?yq+^KOzl*p)xNAss z!Ld*m9*!x3JejI(j+?@)0h5NML5dynJC+}`YFw|hu%gX(`2IA#Lg)NZrsObX;8nPp za@iJ`z2PM8sx`3Ty0Aflg@AY{YsWAqRiv3hNqGIVJ2}}4og8GjL&qSSkt`rHfXU{g zIFQQbqbXqiW-`NqGs7z2fcE6jS!(oJ%^x_r&8+Hjfh2O)(VVugGKaTE4UBGONxn6j5L;r0N10 ztB5kXWi~8-1B8WlI2I^*a8ZXx`L;?khaW!Ag0 z)ck+q9*ATSBO>dQ^f>5q$Dn0uT6DN-<}oiuNcW{30&s zPQU&EL^j&c{XstnVA_tq0@usZxA1|G!xW5W7`*O(R3s~>L!feCobV|sSQEFnWZCMo zm?t_nv&XL4S`T3G<-*g(YC)4zBSD6vj*<+@44y99Z zCg<7TPuRa2iGtzkkxq}O1HjssK=!(|r;zvxIxoayE);W)M;G$-9=lqIyng@G^dg^&6tsN4@^*GJLtQNz5)KQ|uoE=-~L`O>_KD9G{f)1ga= zc;$KkO;RKnNa)|-LB>F^e#zxyk_w~xTXH8=iK==##s(pPW&ywNOO!UHfFI3kb`UI*lUM$oPW(I^izKM^5 zG?-c^|IxBthAJNT=#-1bWXfm{i~Z1!Us~vW*QlI=pLm7S@(IA@fT+_#Kw#AD;Lhm+ zb$(e;j{(-q?SX61{&4#t6}fyO3;Y#7XBqMMj<5A1V|gMFiyIgCI*(TnTb( z^|m4WTcnRzfS&3n>ALsL{%=rwQm#9MF-4Z<1smyq4B;BH?VpVPkCB~iz1=>Rf$s>z z`~xN9K_R1sVw!qch&&lzg#S0Gjk^q@^ZbwWhx|wS|M&LRYYHckfEXG`<`(6@m?RB@ zCPhuwI0(Kz4=b8)AV8v%Bv8mBN}&lJSVuaNm#@DR305lMdnw#gR?c-1Bu#{PXlV9) zcscp!VvGOB?;DgU5FA`FO45VESf>wc9|Z+QC@twur&3>JHh>yh5*%jOe(@MXmk)kQBFzD6w>ult*VxsIZ5CD@-Ok(}C5T4xzE5S6_E$8U1yDWnl(UK8< zG+uKfOpfxFWQD_0YS8l63xp_o6&`_qAfAc^=3eRN>~qT^+|!irKyZY2u413dtky)Y znMJDCD?AWcXfJ?c>jvl#uM)mBEp}J1-ST}dBI+a?@2vjbuCxK^Svc(|&}F3J>q zOZ30CosH>{`v=X7R^`Lw(iIV8bNqc_shA`7KrZ`8BdmbYe}BO##TY=vUS#%wP5rg3 zdcw19d)iT8L>gDDxK)?s3f+VX-{&UDucU>bhE~KYH0Kd%52cPVz$;J>B$q*<@PUUM z(EiTX?hk`Vuhh>-BVXDFIQ$}a4{?(v|5I{0%SD})e*sC-kJDjTmAY?X@N)M5-<84r zw=9cj*dc#{0Rd@EtRdwC8sestgT|yao7zxTA_;>S<2oR5cp!(89mXMQ6J12P7_n85 z%5u#Y7O|e$truG^q!d**8;CmR;1}G^5_ZxP_=GEdUtvhfOSd&hcfRl7`M-dHqbhC zMVmw_hq({;x@fk;-KjlVB0LdZ`Ovv<3C5oN+B(%4ZvOk$?V8=+Fnp&&*w9`IqVxnx zx5VfZw}$A_x5(hs=(OaikW6_7^O3RTr6U70#kXOfPo+8pi4&TP1TXu>G@Y-ln?Ru< z$~$WG++kE2zQ7q^xd%$F@~tPhzCsskWYvcAtA8ammrWE~8!Q&W-1tFS`3KB#htbP%PP9tm-EifAYkP$|qir!kzS| zR9}ti;{gB^*6q33HHZrl;nDS(U~{RVFXPwME)Y1Nn& zc5!eZ)@ceX>Ds!vd_IlGioTD!qh;#(7N{F*cH!j{@?&Qh0kz6_NcPhCH{TbQM$b*6 z7+8{g;FuVyHYgzx_7?>pTR|$cS?|c3!}MybS)H?IQCPHvhr3`rFu&TxfqF?jwfF>Kieztestu{ME^|jnNNU>_UJd`0gyKTqE&lF*Q$9?sVkP0y zuP^m&W|kwKVk|#2qgEJTZ&e$BvRigQwOa%_fC1`dbxZr%K7RoV2VkQ;jo}Cu zV)p4Nl8g%l4*3?&+iLuzT!hOxs;E)pRj3G@; z_+Jg0+9KW9AsJdGo7qw35CBrldH5jb7dofS@NdgpFppd?gpsK_fi(3UVqQbAb2WN$i1u9^v5DSM4 zweuEE7sa_~AVomFqQP&^!VOs>vzJT(qfIDr1o5an0Wvp)@2L!pXzR5)-E~;&sx6rW zsvTbJ(86GLY3WQfK{@(;6t%Qg6=Z9+I*b--qimVC%rk%IW@O1OzC^XNTC>G5`v^~| z=h)#X)M*#R0UBm`xWSlV4KyHcI<1AM?|@A89Sf`NxoP#lu&2hO$#v+%V|@LYM9~Cb{#Bh zs_xxtUSSD|{}pSiu2XBRyqy)*>O?%9Wz!hJX$9k8QQZUsPnv?}@-6VY|4-kU?nMMd z6t*naTMD{P#E_mlq2USIIK^fA1B+V+3Cqgg3roF`7I3N%TX0$a-4w!Vk}U1fr^&07 z+9Nu5{6`pFBo{+XkBe)ga*?A$l>xlYTM^AHLAUF3$K`C)hxA zBkP&iy;s5RwXE>ynJ$v*g_6UTTOIRD(?hgzxy3cl4a==O*|#d5!4#8pH)+P+q=lD$ z9dlFe_A7kslJ475Lw?;TV@X~q$f=1T_v>|8cy&~NcD`a(hg4vP$IoBbXzjz}w%K)2 zWAiBX^TCo9i@){`1>7SFH;!h_!B5pEN?JoMHM~QZ4{e?&wE8$P8TB`iGr4|md<>nf z5jboEy^*xe;cZ-D+dG%#hru4^HEpV_1Bcu()sjE@2rcQZ=G%b)lMM01gGZ0ACHO#% z2q&9gyZ(fMtB64Viq=f<tK zq7@cgjmV2OD+Ev%y!{qDLpapP!F_^=8luZP$w?S?xqK5uO!*fysZa`q&|%Wq$>%i$uDws(f^G>6h>|_$W=Maz-COC?S{@!WWU@` z^hn0BivhGbyeL9pwZ9`qGMl7I~G1I49<# z6IT@Sd^w=XA6$n-H4dA5S$W`3)EOV$1BoF<2xi&f%!uiiF1vWekxtmKJa@;V#7a$U z#`EX>oI26s+BCe+7_!Ib zEv?k*0%{Gjb~R?$LNpOm^TO5$VfRmPc)aXO-1^WKNyG z)B>aqo_aCJ1ADK&<%ysdChzN(w8`#mZm`r{r>OmkqA!Qx(-EV@Z@duOTNhs?%KLIz z2f=87x=078*YXqv{MuK$yTRbe?Z6qn2KONTLocR`^GxzRB_>%DFp(qH7NGqbjy-oY z2HyyKKwB#5mwsCfZ*a_3A0^Yqa?^sM10=A)+q^Vh>)PtkwaDQpX9+uM3&q$K5Gy2W zu4Nh@{F&3xan)ew7Le0mqBr5QE%s}pcI!I;zBKZBYgp9}Ut2pJXP1$&YJQdUn03S| zy=Wr+Ot8Hx{nzSfkcFw3{hm8+kM86dZTieSqM#H^!nxbKIYwLK9=S)^)qq%MlqZlO zOjB^Dljka4%p+&m2i%<}un5MD>Xwi4w-ksw5{%v*UYo#oLRoHa_E{_9+L6GpU~WeL zb_XkX?^HLMmnjkD9A!fp8RY>z1Vy+gUD;V>d6V>)uSl-sr=_Ze@)^B!!wM7z}}NFBnSvJQsNy0PNLs50x(<^LJj8rr`m15#OyIc%OFG(9OtCc{Jmq(Sjb!L_^Hp#`k@f(rnRt=32 zs)SNhlzP=hx%_W_X&B6S%{RUuAL#kR^ram~rmfAF;m(6M&&`K#_m2B>6R_u&)4tcG^sfY+yo;fwp|5Z!+V)w5 z)g4tc=uqO4(x-0v@HgN;vd|s@sp=y<1Ta1wY&jfmDQdbI^!OVJMB}M65bGlatr~H) zt~xStvEH1#0niKYMm#cqwdSy7CuQ9rS){ONHtas7SNA3-r(4UR@h;Sf>r|AR;ObeK zl;%p&7HSOF#b&hSS4jWypKd%jR{Enpqa}78RB`eUl8FQRzOAKrK60@ICx4#VY=h4r z2ipifO}8~&6hdtPJSjzbqL}JmhX<{|^*n2V&9T!|vMaI7+_v-oK$V7O=nN8T7}sQc zQ;1Y_kAVsD&Ze{X^;~a^?%RJ1+9Q~!G{IY+XQ3--w;uQOWTum+{`a)OF{SO5XW-y< zg+i&>5g&@MfKsrSaaxIJH<>xDE=)K+4h0*drgrp>c*&g(WWGPD9V~WrA9whDhq1v_ zAIx_WWc$Y}CE}n^T#(=DacM#LA8=@Kxy`<4J}DVWcjOLi9h2Z)rQ7K? zhATN67O_5)LMw!qmzDkBi7z^MERBau-vO8|ub^NbDyX_@bzkg|f4H%jn2-jC&9TiE zZM#d0dz}D05agaq*+qBYS*fd2i{*SvQ+V|O z#5+=W`8~FQjVl>T53T8!-4PtWfLtpNF%JGjYyOc2Ak4Id2f_@jNc%(kh2?ATCSRnc zR~1;sFn8flLVz>Qv4h?KZ0o<>R2;uxO!m&8RQ6Atp1E5XzluHg`TqY(+U6-M-9X~` zA$HB8qx{_L%^f5-aPb!ByKv9-Q*%K7RviXFM3la8fmUS`dgXyH`k}|VBY5oMfxtaS z{|^Bh=+#%}ZSe%}nGd)Fzynji?GxP_y~^nFuHNHdD>Ye;lO>X9e^u-X#ju2)V7~JP zy7oHKQa*ewW?g1P29hSTROfm7=|S7rYqa8|WQm>;NR4JJB(Qb{w)02}$OS(L`mUGC zN5AdU*lx(&Rf`1DG+UHS^3s&&pwdC>&}cME(7Ee?rf8w)NX@I^ z*(4*leeUe2c2g5r{B3^Jk?pqWxReGai87R?2M$BG1tIvo*@g5@g*7*iLt+^8RBb+_ zKY4dI`pt|i-@M^GFbanC@UBWGYa z$4--E@IW5@k>bcpwM}~Og?Oy9J;9+jAmjsiB|QtTN%JXd73Y_=Cnfm`YheHi@MiHu z7Nex-3S8d)VS{9?3QB3FT}`U}#N#h-#i21Af?KX+%(I2K(m`@!UTt+kP|t>D+aT~q z;fr>#z&zReQ;Da;8(2pE)u{gU7<90QQcaM!d~%ypZ=sz9s!OgToQI+Yo>@`!=JCq2 z9E5}jxm=B3(VH>g%KOcbdy@l{;Bg|jK~L*r_~Vt^JE7p6?uCn3itqLxFhtVgcCO(gy;IORS|xonWD5&A_Ez9GVBhbhb{F=5FZ^6L%+ zVyJIz%3?!u_N*ua+JTmW^yjb10rw} zos&dv9|R$xNHcx}Uj!FOk^nS<8h~0Dfx*ky0V7?Ury{(Jo*%$cxBL5!Fb}hs!n?2Q zA=HefedU@@Nfd(=T;BL<-Mk!@}{^uh3f2^+X=(<|;Wd9kwZD>J2i2n1sVkAyY!vnK5 zp?y@AmiQ?fI>&BTL5blm!ocbw`ze0KBL#>g{tX(EK-QyR92>Qn?j(k8vZ7bBZmQCz z*IFsJC{k5GA&RixiqLLSwXRaLt!naHf7X7i$oQV|W=)i2ff|k7eA(8W=AHPVdzo&~ zC*TngLH~@;Y<_UC6q8_8%21q-ng)s`l1^d0;awC9O{;ojMZYGCZyp}=VB<1Ik4&L@ zSM62(iZnCA6jD4b_=Ungm0JcV0(Q?j;p>=kPd3C6GMd#6Ib@q$P zvM!F57@U>SQzuU7EX|~qnWGTy@q$mqc(|ET_Bw?We0U^ zYU?YSC&Sb+jo$IGmEQHRc62}OW+RBV;iZbDnv^mvnj9)jNk^-k*EzbKf=6x7Q&NvLAGi0usGQ#r5|@og>g~+9-z}H$v2%+EVMUeXLU#q zV{K<=mdi$Or$x!zo|(b90vxKkw@EDSpSrqqzx%_mO=AOD|7VmKnc;FjjAsTtCm&{H zbCCkZ*4P-!chb?anR;`t7-U6iff}^~ZM*swu;o%c8GY+nzpG7h^Je!R#27aNdw3

xoQf!Sb+raC*4XNF6uYgpH#Jq@06A)^wK@{0ORn*3 zGt~%Yg=~+a#TE&zvi&`6Zgx8H#T;m9 zCwPqj~AlNrDQpQTaq# zM?&~@tFXM<|5mrTo+X<#8s`idyJY6OqVi0#JQ(7}*H`Ewb$&@L=NP#QD*wXVWt}`# zFP_HfbA29pj9f>!$XGy_nd|h~jw!JFvi`JnD*Kv4QVBLjTT64(V%QG$-O5NWT}Js# z(o3qy{fsx+>r60t`yYSlBNcD)yyzo-GDD$#g$qCLzxg5`a1Op|C1s2+fufsva+)h5 zlP$vXPttCsql#Cn;F|bYYt?UN{gQk6SFXYXQdoB1W~7a>`zN2@A`+=cc3Z&K`f{|T zh;jZS+`Wyn0?QZ}n7Cy7{Y=&FV0+a!j&JOI>En9Ay$#jwz#A1$%$=h9txspmYei3h ze$_qWsq10Dce3?9@c`kXGIOToL|HN3_M=xXH!DBK!UoE+@wUS1ZT=@48*sEDxSfeU zA_ynNLhZOp(nP%m`MH@S&Kvk7N477J>h5Nxr;aTZ{{5xSo*QVF3>9HJRxDQF*nsWsX2VY2#Q+p| zb(F9Jb__tQw&342$gGO zz3Hne#z8H17lniy;6oUC z1of52|I(+B#T?`^p{ZqHg#fMqt&TnoAb+IaCoFIznVh>{qUt9W$I0OQ7HsSl5@i)# z_DT1n1%;n--1bYTXq+`U_oO0zM>H7p2Qy&-B1My?hiakN2k2TBQ%2=P@nc?T8jGyNxlZ7PmS=IDid{!8_i*|PRY4W4RtxCC{0$bb)4`Xo#rRoUi6m7sZu;V^Ab5%%v0U*b7YqcLFS=o=qOy@5lePN z&T9)`WN_s_?{KiYZ~ce_#Qyiwn2}^@Q|#pk(qp|*0Ng{kfD3!qjW`Zi*Ycd^8ODXF z4J)`3@QtFO!t)Vj_M0ti6#iKah#K*^nWO7#eDkO_>0bU4n746o2|9)qndnIXp#>;hJ8C&H$hbc;S3t|E0sa5IrQ%U(LZXzQAlWW zbIuz~0^rg85W-~>R5u3$Z6&z-okXOWgk&%RtkZN#ko_tw%+b5)V=tDKo*)8iKkbLT zkrS*HtIW3bDf&*5e(97#MB1cwSTL5}+T$zK;!3@}ZlPPz%Cw%7GgDNWXvU zf9)BZOsj4w%WgKuVf75}km>D0Z)UV-3MDpsO!WD8qMf2?sIzEO;xHjGdB9+`u z1!-1tqs}w00o3Z8PXa8Ds%=rHl0-U<$%7r{Hj#V1ap%E=_*2GTf*^R1PebCa1qmMn4zck}^wNoTd1r_gt(W#3KP>n* z2paiR*IwO~_}lYa@cw*qamf5;Im%f#8Zr_{&r_av4Cd_E8oKtsPKQA;FL-~dGlBAW zjZ>6zTOKieo6=|A7K{Tc!T$_KoV^qgO);MmI;hQ8Exg)87k$|}OFamwMae%#HC5VU z#V(oMU-{Od4@J1)X}j1`+@bh zA4wmf03TzrFgel@9hN4%8H(_th`?E>&Zz1}@5`+u<+HevG%3pWQ17fRw&t?YgGWxO z2x)Jsmds(^{}46&eghH#>28yqW0uNTBoZ4W+y< zqpStZbzV4p^V{=W!Kt8n5$Q^fE2}Kc6KA{SypMPFS8=zz6c2(nLr_*{#Z6a&U4{}g zbqU8)T#HiOl>1Tp|KjQ#gF6emEuW4$>e#kz+eXK>la8H#Y}>YN+qP}nnaq7>YUbXm zbH1IY>U?_Es=fC9ZSEobq7d`G3Ol0lO)V=zFZXgj1;whWOuQ%)@y>4F(0%e?fRfC7ypVxs_#H)rBh zE=vOn6YvYvTq|-B?SnSIf>K~5yu8Xb6p8V?Idim4es-)>fNbfbK^u;wpQYX+jw4wD zQ(V(5Z5#B*fqcZ(5EPdg=*h7o+(H{@`_)i7(lp5f^c%^9Bi%_tTs{D9J_E{Hm0T|O z9eJ%E>XIbwjwEs@Tu|xNAuU&a(UjTKN9L;dHzN8wO9&(FuZ0L785)_N<{b;zi7hed#d-I_+?7*WfEZjGJ06V#xYwwCloX0 zivoQLza{*Zmhiyx@S}bQGTap7o*n}aEgq9EL*L-NjvRP}!-)1>e#Pwnl<8=E{4}G?>ZK8MCCEQlAqlUT_ei zqp`N<6;_sM$}pxklVzEJET^@-$i<|xsp8)V5RluHpaQs?8!OQ*vW`BR^EkT0ElLEG z1_NVJqRh^e6-h3~Cmbu-t7y7Dw_vuPM*Im8Llj?PzQ~h6U2Sv~G1`(lZ#VaM%vc2) zK_O(N>RbQ~Gd;ArD6)mr4zIkFlW#}!Y3M%R{%Ji52gNCsM4O$dI1P1%zr zzoIRuXa_Lt!DHGyS3kk0PiP+@wgh+cJDC73L>Byf2HMs)~>&UN1_jmyvu|w4J zKv=;2Vhi5+icS%Y0E)rdNs*gpnz=+Jt~Q;VnURe@rQbR3v#~tkaK|00Dwsb7n=hyB z-$*`TJ84HX@e0*p-SuP#^Xcy?0MdhT{;GsL)&ktzw|$3 z0~jyyfnmtj(U=8uU0m->@#9GNtcmaLnE)$V8)s!Y7V#4sHoa?f>buyZP~II@#{8su z9P4G#7;UFPj3m0RDF z!(E@Xi?UwE$R@uY+3|Gz7PJvei}Y(J$2gk*lOqM@0$J891x|9$M*l_{=|{;eKLl`e z|2^iBEE`lHb?PPm2GqbSeD=jf?iFJ7fMDho!TtqF?iEpXj8_!5%#SyJD*eXv36{1N z;J4%TMn%m7^?CvFhUFYkr6}12l~6I^gmMD4QazB&GX2N&BjgKOF$9uXu`5Rb%BwY601^Ayvq$CdF_@(kgZWK2mQ#`=MGm zs0G0bc>ZJ7IYIN!U?F*Z+R?gg4gyJ`N$g^4`85$x9W;T3CZMR^{soZ#$!GSg^w;fM zv@pCtO1Gc~B9B7m!0<}Di1h9hki#&#t2Aja1YUA9`<(_h)mdaq;6ikD{RzmhZ_Pec zpuKo7_dCU`K$M{p864TH`m$d#w_%ty>a~=~okpZ4fB1$pB>%Y$LjF#H-cI+n9JU;~ zj!_2>>`oLYTl$G$hIbcq!GvcHH=y5h)V44plw_FfCb~7Egk(^ZW+eU1i63l|cD!hh zj*rU3fn=!;awf)Z%Q3;mwUYK=RID3gQ4NcOPN?u31mn>J3lGmU36Ps-r@9z3o2r;t z_iW^_6=x!=>07;+Du9`ieuo&#p~fY>R~HGSY$bs1U4C=Hkt@oE|NdQnCDjHDX={Ul z(DVxkNH4xtRsv98eyXxx*10N`$2;&?C-SzW2oWX^(KNFsv`so1Bdz7qZ5 z>J!W-n-Zz0dl4tO`z&xQVeTDox2Snds@WQGB)c-3l?51i=8AOFIemy6Y`*E?Pp5@0 zMBMY1{5`~jd|bPDGM_s&I}g~wHL{javnu(E3pbsn2ta}%4v4HEi9^aw3?-9Th9IF{ zT_YG@F5{R`PD1pPGpBqMAWTWq+?q^idYy*%)?A5w-AvGyYi)7@xR6}!SEhydSFg%9 zAFx9divyluX!#+dfk8{aQ--yG@EmxTAm6{MV;9vZLdCgmk4@y7>;-d8RMb;h{wRN8 z;C`n93+DTF&OiW<857N_xW>CqIQ!<)Vz+o!3$GK&J3^DKVdTPJM(yZ0Z1v`4>boP$ zoiUstkBiG6Niz=_gu+#rvX8h2E!Jku>Z_^V8cFtH-dneY&HjN>1(V#Ic%?ji}KBsIAxLi!ASF5Dg;oPsgir=p|T-=@~`M&Rm_<%A7w~4Ur ze}-asfRUDo5^P3dlJ$iGN!G#Ck(B}5I0B#?tFFK2D80qEsSu8yDg~y>p0Y>NV*e0zV|JUuj{kDw4WSW*>`Li-swZ?4_Nxxg zEDOK2h1mw&${=*33_p9i6m%1@YRS z)H2t!n8Lh01xe8SAnv2OoQd+$J_XI2BP07{3@zpvD&nx7d=5pW^c@qJ4Np}4(E0)1 z+eqX86vG`(g#__;wHxjm^R^a?ZVAWv2jP%~&0I1h9FXP!u0CgIg!mjsy|03+I!s3K zgWc5?q6Yy%3fv{T{p{6y!I-2sbwR_o0YTA*Dngj^9qS%6mM;i8vo&EXTX+7{L*pjA zmMfUv)SP_1au^8-LD{1g$b8E;(6IpTP*vm%uV)cDR`0^SWlR(L@$jrvPyWGjOmF{f zix(X4>Rq^(d_URbMR4oFp-HVZ`-lwrw|Mdh1I?3;q8&B=396IQ>#0PV;SKtyk5w)O z^5c4j@|}St^%918%B0K`RZ1d*_i4xEKHZzPAg&U#lCX9cP}(QwE6Nl06(#`7#-Oe` zo;TBhHckC^q>+r~AQq-Gu|WO{*0?g|zGuWD8k1CQol`&M2H|#88&iGxawLoC_ELxL z-g9^V`*{CtuRoak1^+&li)9tlR?Q3?W%xHSoxF-l?+sVmlXs{Dk^@vbZwBv7JQljf z-4**Gq+XmdSC&yJ z6hgg=+#NS{7hZv}=?Zk5v*WMDq91b(cd1VHYKT;TeT3^lI2_#K&Sx6vK(7#ZmsalX zYZ$;Z6AW}UC`N9cg~%JmFlC^(!FEH>BEm)pjyTF?R;I}zsIlXs?8B@%bFIO7W&WNd zjHn@K036>n`Hd)X=n3G8t;lH&<0obKEAo$&A)|uOk+%lKN&O2-b+*ob#2%$m%N(Ua zB>LY8?7xzn&akNaUhU2NcgLSc+q}e^yusgaJoHM>r6g?XGL4G|!Y*PyQJ{gy7b)Tl zG^i$2f4NZ#_cYD2nPKpzM&$;EmJv;c+bLe-7(EH8=H)HJP|=JOeM}795C5`ZS3#1% z;?DTaU*UkV#|>W~XcEOy+0L`*DpbsSg{a#L-`6yyrJlAqz%+lzy5}&rWiBCI_|G%` zzP)AK?-4OT;_<Z)?5bN$L1K~`b8UM#iIJkJRIQ$1N~f=g5Ae4sA8NO2#k3UPE8@QJqe(nzw6 z#CuiGO~LAcUi{7|6qn`;?FEXrhIG>5t1D ziedmd8(6#{Ip^9MAXPtJ2sYM@pqvYhiMRs*fQWUOwU4ylJeRn`~YIfPgIvi$o299LzOxazOE=wZDG+ypLM{vg9q?If!CuD3697!?&P8DyZoQyeg zwYs{?bG3;hR*>&EA9(>=ZAyuhwY@#^nPj)R5s*(}OW!w{k6)@M@+SE!L3@x)ay-I!Bo zkkY1PWMsga<&9>hRr676%Q1q*E4n`4T?*}9)I_8i%ss!!dX?wgeDh*dZ4Jtcr~S`m z>y*aO--_OZKE>EUu{wNB5!fidj<9q!RXb>N6s#9Dy`9ycR#ug&3k-#u&@j2%)Uatg z%|Wym#X(iTi^d?^i^`7PFKT*PF>g#IeU1?hx-okkgg)7ab*9c{Mzdq8K@TM~`mn*owtKunwzb9aj5{r1 z2!%@lsJ~8zfri0h3_{4k!})ee5MQjZoTQ$2Q!&YV-ibW@pPP2`bG_ZM#t@m__l5ew zm~YTl0yxhSC1>9Cka%vbOQhD#2;cZE9e({z%`5a-+I=_kp#R&GOVa>75ZwGvZHJat z=@cIQ5xf%&+x(Jxr?nlOQmze(o%0O(t>iNR@YI_#?Szx>0=mbNW}b5}4u?#1^lbG__%X>6zaWNNP(B{t z&$*;O5PKdO{pG~o4GVF3bb^;1@0ACD08}u;=+w1xy^J#8>^|c_zh&G$fP#SGG7&R* z%MBbUF5~>UO*zwm8DUHz!0}%Xfh7PTjScxNxdm$h)KI{Kwfeqm|UO}jMGy{ znd+JDvp+Ad9;df;yMN^Xk8>9E%2=5%|~LlPVNQBUxomVk}Q8OJuQJRF>OAwT&=^@DaeJl%alj4G*dix9`_RX3%Znf zk5Ju1KTa`#T$Vc^%H)xsMbdFqAyv?y&?UN{59l<&)nrw1RDD)CY}9nZb<9FN+g#r- z!9HVagd0(_J9>g2YOGso$VB5cSe=It%gI_zuSI@@H@yQ!&7HYrsYB9j)_UIx<4q-v zUoXscZ7^xIuLzbhm?=3mzN|J5EyrHVjs#CFsx2HHRqrCmxdNE>kIPF_E#NF zyq^ing!@HAsLw8=nr*usPp-t2(YU3G8FC<&(IcpE1CJ`+y1XK$k9QZl*EF=Nm7kl= z7b`oL(`^Ce;6{2Wy(xliyKld28_^4<7#|W}!Hx7+e7IAAQwhYeV)7}JzF(E`hX;wQ zSfegJfkp6#3&LRzsyAtv>grWCm9 z<3SOZ2B*j$y7(msCxj(ODZq?jBnoM-`zLP{-=U=R0caE|hJs`y2Zpg4l%iAs+&D&L zf%;JzXcRK04w0C^pIg;0#UdsErY7h@%;#2CIZ$p{FGxFYx}|71EYKG-it zBny-O-8}sNBPiqQm7@@UpN0i+(eZNJJ zIUsNf=e~U3v2DtCeVPd9eqr>fd`XTFGvdbVGlVL;QNgfPf*()^L)@AoiyjBKia}f_ z+|q&--`apR{`6@9XC&U@f`!{PC9!Xh_rJTk^67p6!O1Z`mcsw=YUe%g2z0#I1590c zhx_XZJOhrS8D9-M-drKFcV6SUJt+f@UB8&&vSV*o;dzIlGEASQsal|Q^-#J;%IR*; z#tA$F7XeuqIx!6W_xIA?fFs>VAQh=oS*sxMN33=@J(Jo(C6l8ukTe< zZtvAsS8T3tR_p*jAkR|opPFFHaf`E}<+R!A?Nfm@#_BB(BC2!RYDcBc*v;#yJ(ZmHbWftSz^texQhWX#|qz!awy;=_iqmc$N0o) zlJ$uj5IgHNF zgu=rUop*iHD`ajksro-mQpsmS(%}c#%Sun){duzuMI=_?N&eeVwU&}{C?}Y{V4Rc8 z=1ncbBWGTLqY9gt3!(t^<$uwsbBw{v<2#)mBhKyiqs0Zm&?zPKj{0*;ie&DM86MRp z)v+uW9wb&JGjrQcSa1?CCmNk(6m3O{KfE^no-F#2TbgFmy}OLUV`iys+XISPQD_)O zBMihOVKz?Sy0%ST=9$S~qv>E&%23LodB`Xu<0)iFG}=NNpT7gGnYb&f5feJnL3Yyl;tK6#os-l6MUMKuk#%M?jAGwEd=lnO$(#V=eBKu}3TRzr za(>u#0f#s%9e%)-gTFvyEw6jNJ=W)g@@81()jYa!@pR|9h~&AU27yDT!(|&y0a2Gc zffS&IKQOB(kDZXfHIBeoE4L$WfR(0wdE_{>(Wp8OS^Z*RuP!WXWq^>F`TKf}fU0AV zl0c*P+G>5{vPiLAi6O)`&aC^)WUbj1UGdi1GZ@ol^a$W9xWr2OXizF)=HwKX&_iTc zAZ^s&_7Nn$R$#rrmWTfEFr$8REDFWqk0R{kw+LEPw71pH^z>b#*@1zHgRi6y5tc$3K^-BXQZz< zD2E+Efdb$&W_=_eN-iKji3-SOcsw=+{r!R_{X&9t=GjK#Rx?j+G}kA!kw;kUiEWV@ z-=ECh2E%G5S zh<8hd0yvUJA6CEsDBP-k$P8^Oe}H@^)KpJLgaCx2GZ-&IM zswtKo{NwZ!RdWvudyg>N&>@ zBgNM`Rur|DXB;?7SHC*@*0nPSo@}`uy)&bXXr@$o>80`y>98FZmAQp7-2M(K=qfn5 z@d1t`A*N=r+%o4AQ91@S@D;y`hy!hK80@3iNv=29k$-O#vZt?9uG=!q^+7j+#!Jhb zd;n7)xCC_J)HTij3eScqv~77K^q3o+MApus2e(#Q=$#vbWAbKhMI0R~xeos20IGx$ zZ*9@>S}(y`6tazq$4Y2z^L)Moj`hKrzYSoLazC`z*ABlH6A8>!6Z(cno#D&XVBXQ$ z5Nltase(#3@1CyL6?li8_C~qn;(6yX>)x)nE%Hh>?S|F2S#E%J7)E7!{FjCncY*tW zXo`2qsUSp^qmMgx!(A3+xO>I{dW3+TB{YeBfbnZtF2BD87ZOwX7;y_f$C5!F6dMps zZB|*U9*{LS=C%MEoY*6!Mz4hq89;)8~C6DOsHY1tX@kJ=m3$!_ilJ1e4DXQ*yPByS?u8+Ck; zvBR%ey&^-XpBgi>UU^DAo?Y`YV-v8Ou_=KYHenFyuyTQ2g%UwCwaR&H)ycm84>!`Q z?++FZ+ra7{Yljf+R{l4!8|Ey51t?f_~K+h)Eg@VkiPgwA!$h zzX@V2pt2$;Q@E`9H2SqwuEr4JBQqaI2;h1>do@wWfHjR&9Xz2 z1u;=$Pl*RUcjQ7$;WFSXK)=z7>5*rbg&JR zq=?J3$9k;7AlEOPj;z=>(ucf?C<+TVryaP^G@|dD2a+dT3`e)@(`i!_WMD_ZKd50Y zG!^fgn>PRtmFLemCxHey1tL)960h_YZz2m}BfLFBt}FvRN(g_&zVyqBBKIgYYr24C z2Cd2~BA}mlN9kQkT|Ab@3RhyB7B2*~A*F^7@-sh!fs3V+jR!NWL4;Fc&f2RGkm#@wOUBKc&WxlNNX^hZtwasUWFh60s#MX}=?^B%K|&f^3tqV()PR zbQY?<9NI+(r-!vMx3-D9hVKAgy6Zj7$v~i%{Tu7mehMEKQ1O*kLI6!wEM?>`S;o;G z`a?uW@m!Reg@A$qDXB&Y1$eIcjvqFZ8^ZN8`drb`iN&^&m297d(W19yW|eHL{{zVgLp>l&JQ}zrusmra^l2 zP!s0{FB*k~3PHkijv{6J*;v%s_MqZa5u$1OC6nnzoO+X5&E`s)dU;NrADC0wW0;3FCs{Bxm_eEM@7fk{iVLk#BEQF)+`^lh&*Ip?>Hcc zX;kT4L02UCmDE8W02I~sILSz{o} zQhM5bDHL0LvVAI)XaUT4PgA6}2=47%Alp)MJo};crkZ zOC$YDt^g4(a|ToN>SuK)O^5TElCo-HPR0!>4Vg(N);m4{t5W!&zhq6C513?Hm(p?W zkezsWwsClZvGF3=4B9#Yos-x>wDky&xuk*atT(t)RT`|Y+_z10N>u88X5QPFPPFMN zTkLA91`y5dPHE4tjHTUJ=8IP~iH_~YSqnzd`2gAC^{)C|ajq_A0RU!DFCQa1of{?{ zVq#9J1(NS+c8_;KAY48(I2cm|!%Z6sa7)0r5Akgm{!!=}_o!){8;+`ls1)&K@v8;( z4NTXcZ%44Niy-0ngi8Dx2c5AbEkc^HF-$a#yf6D>>O`}HHt<}n5OE#$7LU79zfQz9v+YlkVP zy$yJU*slSfh}G_xm|eqe!xy-N2}cDaZ9HcHkgTN+wW?z(2e>Q`mUhV*L1%RTw~~ilz&aGWuqGQ6_V$ zPDq1!uGRsexgb_!Lr&#}2JN8E%o1uC#JrL8@1j)t)PD%qE+4-;UWz(y@M(RO)yEIx z;ksVYJ}_vLTc(XTQ#T66gEQ_k9lGkf*S}~+OFw!Di`pxJz$Dqtb#N8oO`!cG-0ulPk*hfQq zk%e?Nx=6vLLuE1Ig6@FmKXodE=4Y6m_$pOWzN@gEYfPE1#Bi38x|Jk z9@Uj{tj{^D&h~fYmZ&tflpS8YYP#vH9n530EeQHjrk4YrhYftMzw46JC+Nq#`b{)g zQ2$DyS-Jq~2Oaw%*Qhv|I?h*Rm5+w&X>!}=facFJOOGx6&l)CIE#A%=(p;*>$j4e} z30n0*V}1Xr)SE6-)KhHf3Z-=}di==yEX;DQhJ=X~-@2*uzp( z^7Ljra312@Ej~MXbfp?yGcXU#>m6~ol#NDnBFYD-=iyuMRv$9XYvksbsE_~c$h}jn zL{NxILIi$yb;2pggjIEAR}62fg%QCxXXvN5eL#|bQb9Et21ler+2}X>X&amj!-8G!Hu*g>$Jj$M<$@%9J!P2UE#ychaa-!%Gh^*v0P z7MAqgt{zfpqj@7VRIwa{6<;r?!EzCF(YQ|P6v;nf0b!Gj;TTk-(B z5Fufw9v)$%WZX|7eUM~+%3fU#11rdvQhDnH z$$vN{E|jq75Rc4!&__n-5dKaNp!)!zVgh5EbHUDR>yJSl&rLxT`X{u*yN<=A89j`$ z6W@nJ01zsWZ_58-4yIt4gi+abs=vJX=f~LBQL;XB>LlExYp;{UEVsCzi)qEMvHxZt z4=oUn^f9;B>zi`sqa(vExr$3e3%W=u!wyDp=5+q9q5nTdYf_gfG zpE{2n+mAF7?YiZx9WFQ4ztd!R3F$GE`vr!BS^VZkF5lZ(MBd<(+=jKodF zEBkdD)oQbOBuU@Yd`)Y>ymSws=^LHOjGk#v!G^ctxruP8C)DE>@Dqi=A`yq9Aci56 z#Mg;**X|cgBQ6JyavaJ%5D1?0HDj9A9ZonOLEyvTv{dAl*=PE5g3qZ8Z=Z%qQiEjE z%b7Zi`n2T| zu#P=YFkGgOrXUN7N2q~xi+HVdlRJuA`m`8)Qs&zt95U`iFy(Id>cQdOU6#z``haYohx%4t!;X0PH}HS2>U~^>RTutrmmL1Q;n>98<#oxN)0buTf;jX`z-)}){hS-JC4TM9ck4zl4RKg(ZCzvA_ z2D|1wQ?GkC>HgseV2uUX*X++G8bJY&Y5czfo!{}GdCs~!|IGh6t z&XFX~6tU@8j%f%!RpSfrA8s7mzGE!VzoB)TP?#k}L zZUZgu7bLbnHXM5p2`D|GKp#=mFg8Yz#(ASk#E+j{s}D+*JCJcJ%ppziOG0N_S~m6~ zDL*~SutHUK2x$mr(XJOoC*|B+{9deCr80IG(hi5|VlWgjHbahBoqRN2^rWRj3qhK~ z4AW9ekdeudlFt5@ylZfjSeUk-rrv8l!I-C z8w*Gq6GuP$or{x@zb>z0as=zCc*nJ`#+g=m2iJ0}3VC|H zKuys=f=Rfnbr_!pJ-sL|$}-PWgVNDmxC-D0u`dWuz--6;#{e4%J6=F~7egpr(*hk_ z8^SRQJ4tS_;$xh+cM>|w)cQ6UQAf-)el zAC{pc^S*3wMuC^Xe!RTQ;Bu43)NW#SgA;kJJ>O)KqrvcXrnJeexIE2cG4GSJe}|*B zPfn38vgbRo?ZsSvRB@8m*& zsq&BD5UG4WDT1Do8G#P;s5oBoKQKTIMVj=0VT_!6LRdsuIF-710OPCsem<3ZPa#kIa zu3O%pZ61Ek^aP3O#itTbQTZ+m5Kk!vA#WMI_+s1 zO7MAAL4B$I`GgS;3EvP9qsYXFy6V?8m~wPprC#Q~Abikv#R!wk4#4dMGfwGD+t0;; z7^YoJO-y~6n7G)y-!C`g0h!pX`$Q>#m`6CpGgsxC;fG3qDAhPhpQq@L{Oxeh47V6X zVC|P5jb|w*1-LCXR)(Fc$+cRoEEalYg9R<%h_QcZ2gTO9KHoA5*^ibZau>r;tv||W z^6sb8s>>#HK)H--aZ18PN&v>q>5Z;CAFN*wBClNqOUDIQ>_-nHgJjB_=bIqpvZ&S|#h-+q_suEkiZlKAd4gQ$OAr z=b}Y^Z8pyY8knK7W*Vv|A+g6n{`BdDGWu|sI*XPp5jGecHuNLIQ_H=^o2^h-RW{d^ zxG;udz`hA;GyOEZTLrjL3j5S5(*$le%gKEtpK{P_6Bf@!7nGeh3?3)T8^A2Z4DZXb z&D!qt{&4ZkA4m6&(WUuPNXLf}>OrgM^iQD}|0A-5_RQbx*uvlju>x~uQ|YGPlhob< z2FZW_bTU5?3UFoThgbINVU5IE=bT1y%L4lr0BRzN2bJogS4|@fvA4f%s5*Aav+eGIF^_b@A8o%aA&Ak zm0#FXEtv2s%T-xHotp?PuxJ<&jn7aG)P;3>9u*suOC98?*#yC;5@D2XWT$`u8!jrK z;zfsL#DsL#DNF~JZX?Sa;=SV(x?e4LduFnWJ@NxSn1A?fW&k{7``eK!TyJ1-l1TW$#IhI*v^R40x)A!B}Rm`u)#;zbHcUMR*8kiT#eT zPwkle--iB4b2k%aEk;nS=Vn;}RZ^Ss;=IP9c=eJc%i_iuR*|<@BW7D7(Lb4|{E3Vk)(rLi_ART|Y%#HnSL#J9-tV zE0T+UQXt5Xdb z{j_w4x65e6RkY32CjD{PBzc0{VI|4W0UfELOJ5utS8uh9*|N=C*CX$=lAC?m9)Mwe zC|)MnQj+d78+to5z$|3Orw)A>+-Bn7SxOH783XdiWgFhRR87mXes2bv zfbHPUaB&U}k<-fX>ZLi8Hr1sKHM8C2)Y~j2xPFZFiD{^HIM!V`2_v$_YtK1xG zkr@R<0UZ{-q)gc{ebWxW*UP4GvuLPGb-Q?4p;In#HPF@cxq0!(rJO| zo(Y-gAUT;PS!dLC(*bBu^eE(-JJd$FzGfru$Nh%9#_RhgvAJW{K`6YU6qLb$4it7P zPQ~L@xa_^5t+bh2lgmex`vBJEj@|J6CewJo!R3()<6({FZSuiy_jw*=1xR}Y43obG zG38rGu?E4}bsav^fzF8KcR`mXh}YpfR1O1zt}))?r+d9#aC|G62|s<+mp%8y0$d1^ z5gTBJAgj4I2l**H72p08OMd;m#Sr8|_iPRSC(#j$8LLCE?RJJOIl}`j>&PU`60VTc zkFF^gr!5+#U3g?SnA^RB2w2arHUy3r)|@CZI}(CV45m;e zfPfC=E1Y4C)6B0HDtdJ?{PcK^7PXpz#C|3C1ngjuAjjp)afAalX!Q#%gU)gG)(#SH zrbW;KR-xH5iJAd(m}FMQG&ZNmO|%6H48QGEIYfsed_5q1=B4zYA6J#xn`ZQXhT9F4 zJdE4@sVi`Q5z9THo* zboNS!iGHgINHc+_nEpImS*qEwE~0Gk_q`e^9urT;uxK_3!$$r$+KZ|6`}`kd;dUvu zw80qo``ZcNd)4706u)mlBn%=vf0IzU=-Pis@aj0?uDT=>)E&G8mA5{O}-mvRkr!J#k?Zr@b$7=}1@J-d9Y?rzVZ-g9b#&nG zitv7s;C+{2cMsho>KH5~lXMX7D*sx=wsV|6aiQK67kAC1=Q7ndTen0{TLi@awbrZ$ zTO%q;G?P=a;4W<(j&t{j=HRZtybZB2YyUgGRMEEj@zV|EjtOeh3R^xbv~XBU%?KA> zPRJV`t0rkB=EMwX3*T%i$VUXa2DQ!^o+X2$98b|@EOOw(S=LmH&9jO-YaKi__VyT% z%SjOqx;)9Xb#WPA7Hjr=y$KlH%OKdSI&b-!-zn2<*$EyEqx- zs5j@wpD`R7)f`VDPQ%=)R&QVYl<4~_I+2owEl-pA%*)ubg;=tspJEtfx|m^RLutkD zt#W5Q!rWFVY@x=4?dyT%(7)KQ@)xoU!!5CO4>Sp0|3WXDb8p**Vi+LUm{G!d6+;1x z6B)kL%ACuw{+G)mes8Gg_vSdlvACikU1AG`bfO-oP+H?W9go|L_~btwRtJZywBKMMxh)Ur6jVjliyU8z9L&}k zER1b8AQdmA=+`Xqhv0&qkP#MOwS=xM%b7M?yEF-G{9L6_z6tk`- znbUyPAB*<^v$BKNOnRzy%h@m|Z_7q*r&N#9?}#kMQ$kUeTY%IVxn{sR>vF}HGdNPW zy@$hUD>#IJjk2;~O}=&$;y~VC6j9p5Xpoxkt`$xdotlLB6KI~XLJk{sgzqyt6B6!& z<}u0=Qtr9{H={YaW!tPLDC-=}Bz`t2B%)2TerFYkPF@?7L@2S-?DAk-??*)N?UY2o z-Q-5_*%Pn?2EO_!uewOBHTs#RVa@! znlg~kgR@-=fC4zu)>IwS(@gtYYX9|zH&d5(g09e1)67rGtgO=!tN%OPU?z!HW#Jpt zXFYRGu#jO+Q>X@4zEI9~n9AuvbPpW{u8^g#m}qNIdsARSoI#+fN;etYT8$`Je*40G z)qt6XmB=jH7BAsI&7@%VA=%a$c>dgGez>WmcMY5o%LRx+JL|D$L7lg1xA`1`JsaYI zUP{C`b>rsvR^-4CgyXT-;FQB1-GAv*qB5<~wd|P7A70AV%R~*j}yu+W;4!Lm;eJRXE0YDd(ub$D0^9<>u6-bA zQJoajqsp%jKp_bjZ+Q#3smtNj5=L(a67%Y zhTBUk)kFH9`8|vZvOXN7hFcUdYI}2uFHAP0v?wXrC8 zJcLCK@;yV<+6%}#6k8YbKtXZ$k(V{j({W(|-St&zD@hEwusT1#F8tnU@MztV1I?#* zl&4EjOq0|zjylq6qH$Niug0vUMTF=}QGmO)%q}U~3L1N+b8niX{L&HJ7>qW$7Y$QO z>Y@t5zwn;o7M8UQ&cHUN?T=+7B|f3p(Yei-BFm6E0pT^*qx5fh^fAuHcpF$dNU8zj zv0*uivg}=jTaKXtw1>anzexv@*-5_wrS`U81U&-*VN_M!eFyQ)6wodT8R2%J+yIyE z>{3-vJm+2I5A;uO)QX~swlEw-XI+uhfS;{4YvnQ8J4H~WTl>M|c{SqLG5;K-EM;vW@z%7G z6BJ`3^HB&kM7P&LK7Nm}dkZ6jRFIeaxU2>M@CV)`yAE_f7ubInP2z#Byuu%U_5dC) z;71k@v4j=>C-x5j1%omqT3UD&r0!2FK?&qkFgjV99(}JW+R{YY6{;;)-fg7q!~BaN zV&DLpHxOUK?MBnYFDdZxxT}nH2k+zB%k1uMZ;+J1xF}G2&bSdW2}*J&4KXOT#np(o zFuR2P6yuo;nrVAs>}K>J-IqHlT5Y8&_YEeDh+@H%ItvHDGK+RlYZ?*OV9Ka`<)6uMH zF$b09;?@-bHkkxt#zKcK2c-QuM6mcv?f7l{@8{lQ2vkukaEYAs+rKyMuW^p_d>Z~& zS7!kfN3*qIfvf zwL9}{?@ZO%?&<0GyoXXifizNJDR-)Rh<)U$v^Mhoj{Zp>^M{IB8pptavjM?3%`$4< zsgcz6q&Z3+&PqafP=Z$42E1D~2U|Ql3@#HvYlJ-fR}-N)Nzjj%)Cki@s-G6XbYX88 zLK>fz5YtnF0VJZFy3L<(^FM7I*GiGq$8){(-Fk4VOX-C`3anP<#}_HZ_e8?7ABd=I zBFd5G#@K9>Wuj++3^R8aWB36S?GhhCC&j*~-fZ?m`J(5@W*0Qh^#ZJ9gIVdBigx&8 z!e)msDi2?GMEKj@arXF(<^StufL`5a-zqT#4UIOC&}PQJOaBgShI{4ro8*wYxflua)g{86j@lh z;#Lxa7z-}zc1}j)QH+%(B%He;nKy%7d|0Cuvuq5WD;hpA6{BVbVR1-sUf9 zl%H^HG11&9h=P9GcV+V98d<`8sY10u_GHzX`(vrs@JG}elw?2%FN1*&q(Cob28+bw zShE%X-8dS4x8B-5>Ch+l4W6M7Asscctzo@Pu3Z_&hU$th&A<;y)Sdn)11jC$qI$lB z!MB`+HPAA3baTCAlTS(-(rn@ur}Eu_%*ZgP;;4Lk@C}`Yr5Dc2V(Y)BJ3is6O(@9| ze95Gabz3>HsVFCpOa?8aFU-cP6FI!s>FWgP>!{R~n<7o1xCUiZbzUjdTOzG6wp0eB zo7Sc2S8iFR3jj%v0=mkB^vdY@t?#}lud0K_fECrj9VljfuDMzEGtQGP=mGmw33;*s-xB z70i@Kkqs`?tH1ZyTN5dt$>4y4lNQCAja_Lz1u!aa( zDEDvA^J`-nPV+EnDn<0xc;D@icOh7)>v4!@jBHc3g+G)!R+4QvGIDE<^_F6oEs(Si zNo*C?`pTxy~lgg#YFnaZW z9;>BOf1d1sM06!2qIv&+B69_X2TC~FJDIwex;eVAnAw@Sx~6E@D=(> z7|u-iPrH5E7+0yJx;Z(aLFexHzd9yBMW^ zxKxNI@9;gyiZk@cly3j{6gTlk+it^YBK4~`Jb7wm+=dEqO@ImEhb<1xejfLyz%uXZ zj@(@(VYaYi`0IF{_vo?hweCL2QHab?Bg@6b*SS>NtZWK9uv4@8oW#vvTfF8ff2}J{ z@G^f_F5%sKT`#++Hf7W1zF#sXD7u~)kgGm1`u^=Od~RfZhF+KH%78eqd1l7?9&S1h z``Ta_ui56R<}TQjk;g*!J|^mdu-WSIOZ{AiF9kfg844=_&3@&u^FB(+FVdeqXm}oR zrmgyC)=~Q-!Fnrf&umEk(J*i&_9!1fwT8Z_31Ozi!aWvqx#I0qQz9u{Di_V${-3!@ zTsdm}gy+hZPE;A(>Mf3e?M3oYEN@>SiMW`fCY2k-W$@o!G2SMtRInh3WOy>_V z0x>tq$GoBoKcl}(rD)^ujXK~DZ8=A&P(E%8Y*jyZs(yVW#9B6bGH! zBHzikiPACkL7_F9qzQswsDF3k2bL1)We_`sBFGXX-#ovm)mx2T8Fn3Cj1G8W3p6NLh<~K=udJ!m>U>`IpagW(7us zadqJDWHj#lAeqko`SWNOKag@!9H$Pme?%&93QAM_TF1sh6dU<_7g`@g#|tq?pl4PN z52@igD$H`SY)iQVzMIFCYP;n-*^p1cOr5T&mivxw%3-svrSqckSo4W%>YaIp`NW9U zhdnx;!o1P)wDR%{GB3k4@*wd3~1HE6rTdG+5f7+{+E-q3}f zQh|*Rf31Z3MqOD0x&|I&UsE4yd&jcUj5^;IM$SPhsj_#k)zINpU(b4re#yG;zIV6m@$>zK*-N42 z_RUNxoCP1%8xqnQlVgX8An!KC<;A-Eg24oX7DK^J6P24VL-Nd+j9pORprP}mgJI}a+nV2y&u9(c{E{w>WKClhndzT*Plx2ya`*j+p z(rq}pGKK1irT6d-ND+wv@CuOOyStHoyQU$G6RWK}=qz-RGA!sDu0_e@X@#tb8jaa{ zR9DiFoZVw&GWzp%WdPYqD+)Y$ge-wS>yVTPN`}c-oDLEsW8yUmTTFs5USG@3NP#qD(w!dw43}%R9(KNeq3})h}D@R4E{=w z_2CI-5&s+-7)en8wr41O7dvv&_$s&ejWWdqx5BN+L0cRaSud8dWN50OqM7)7N7x4D z81p81Ja!v(%r48)Bt^LILB&Ck5^b#F9WMM?m5)#KFk$?NE_J6k54Zj^tbJU-(Xs6> zNw7w^xZW_>Igz_T2rt=?)w&d>^&qd;Utz907y(m+J3KrwXIQbpEcsIU)xd$Gj8}Vn zDSduK{pH_du8CrRCM+a^TOn(>9RC?}xqvC^JC5@z5G6f5&TH4ejYWQ3OkM)wr7pr4 z8Z1SHQOn@9fo+!AI+nxbO%JJGw8nmOg9yK%e$nl>F=uVP5zA~klJ2?9xy;^pdAYsC z4}@v5H&<;;lwgh@;W_nsZ#p6#lHj{5%jzRL%9S|++ejQ^(Eniu{B`$-`R7XX_n(2T zrSq>r{i1l|3)fsWQe{Rmbe;GNZknQ4n6%NC#crA__@>7<6CSK8W>!^$qniF`HN3-BG9y@F(kO9B{9`)bVl5MQabC;vbtjmWkKhc$|m6-HB#{h;lZH)+FBcCiO^wuDF%$rHkZIx(JnwuR&`=@aNjiTNAq&osFYapiW~@p)Y2wPdpwR>Z`+f&F;RVE45}A5Ty6yui*hudZd% zPH4U70)RTbY5I;&C*UGb>IQ+f)GiEpto zS3APEF|cd)l@$E&v$AP7J2Y0J%(`nCH{|GxLK}+dFO^^CHFB4lP2r@e`81^YG(bwV zIO}o`i}ouvXN_KE*;l3WXxr_R@oX@ovL62LNbWSHgSrI?6k9M`u?%oR4O8(EbNqK^=?nNgRYX)^S`w7S(K=)vu%#J3~)S1DH-Y6&T|93$zTVz)xqyhf$v3I za?eBdo0_};j1qJFO81RMZ3+(awsfx^6bNhTmEsskceB%$daV$?L9*iLV z5jKAGo(p1zM<*nrg1eq#jZd?ahWVtzQPhEG(a?ctHS7T_a99Wmrdp%YcmH zH%Hx_ukANPFEF8jG#k$u*s~YhZ|p2PWbDjI9+f-PmhYA-WJSNb@<{+QF2Yfy)<_uD z3vSx%&D$@~#hGgo2UFx%_X2f#Fvf2zDnIMqj5~iO6XDZUEZ>qDlKdqVoagzr@h!gr zMeZ8;Cl8EuphR-EPj5MU^h2b_^~ahLuap#;`H@g1lw+vxi*9=@_mc)_>Kggq+u)(BXdL{_7OIcXib+hxPP-?S5n(JjP;$cDRNzg+?Taba@5$sV$3$!sY7)pGR%go^Q@oBbLYY;ZE+l4txWpA!K zR2P9Fl;=sUPeh;8@kx>*or%c6Y~~Yh?Z=G6h%>>44l)Uc4Q?}k9kiy{+K)B*6>D{> z!E`M|I1rWG4r*j1;OBPM+~hG~mw)Xud-d`9}lP^D+T+nNxUD_223F$@*_V$-<;q12>TLX zB@{*t<&5BJgou4HbUVc`?C@f1NNVS%O8qXmw8B6Ls8aK zrtCDGQJpKJa*T`Q)}lR5k6%%Bdbar8$a^0#Irqhi6e5V~$6&;T`pQ1zAPw0^l3__M z#kg{bg;UeDou%;rg*&r)EZeUtm8d7XV$6U$yR5nD`-ES~J#-I~Mi0JC15ihr6K(3)|@}w4@lToL@i)vzx{(6i&92{w_0Co=^n+SGp6v5)9 zyjG!bi_B1@qN{6L_%V&I0OS*G5XRaftjAx0HFM@%PDZ3rP+WE$aaFEs7Gqntc!U;H zm7pU%TE6>WShb~B$n0*9#A!J*%x!}g^T~1=d1x=P^sD`%tu))b+$;I1X6wnx%oTI9 z@mKDGA?%!HJBq>2F_BZ7Y+L%l>>pREJ{oVeeyXoA9a`pgKU=QW*#OcZR-K|4VO4!q z3hiHFZ`|6{?4n7}*Gl{mtg5QNRw67*STtWw9Bf!TW{dnuIymwx0kc73TYK*dA)3_s zNBdW|+LC4$OBeMfwyRYRNa2+v8(Lu$rPb%plF9r61ZdKpJfR*L)o~rLxfRi<4_Z*3i}_jJN60dI zd`cgiU86|XPm=XGATT%vLoXiT1WjjzPbiaaLmdL!7FO$kq&M|!OYDHs0jZCh zlrv_WBcjw?Ru~LK#z(`#oPo`z;S>XLa#-P^AgFPj&zXOWLeWd+6D74f@`j&RF0T0) z{mPyebGJXj^6k;8b=6V~wd-)M=D7RNB+nUQ5+|I^cs24+N_X4F6lvigb)OS2@Au5a zcM3m^Oz$N#68NVwwG8IIhDJ$hqh>gDzVXoE%c0+^I^ePaN?Fn=9(I~OhWdYv`^?g} zXeZh(b<@NYqV$$$X9|V)+4C@$^IO+$f7zI1CM@@wROi8a<&1c_MWfdWubF`$sCiY^*^|lbVa!`u_a4KOX*4DkORginlK}N@e04uSa z+|Y#`lltijB@$_Xt*l`_A=K?PL-;0p*Odmbq4;Wb)u!8SKl35zJh_#;qj)4i{*Xh0 zC5;i0ggSb^1Z}9ll0%}KT$`cLk;o;*K+&sdd8t8f$i6j9Glo;Ut)w!>IS<}o?9@(@ z2``oq=n|{u^ZdC9F-%``81bTAtA_JH-=&M3mri<F*1 zGs~gz#r-Lw1#3f%QC}rYV2QNW72^56veUxGEXzp9%*Rj4na&7+GBRmmtL)5pxgN0w z)GKLvsCT)g1U6Mn)#HJ5vac)d z8rrKlE4w*NE(TT(oF(oUC23Oaj37#`FEtaV{mH&VrHkb*XQ>Tp61fSqCyV4}k<|A| zp9<0EelK;4Yr zx3s+UZAfu*gup(ZLY*TKiXtU*_H!+;Wr8jSFGr&jL{|l6&F<>o*V@G+TqFl{j;fjO z8)}+iC>rB0DE{`&UFS2bW`5<_NQY=vTZ2D=#rQ0GV*y!KnSKW@+H$ak65ZjgGvOEN z8FAq>f#7Q?SMC1Cnm(Ti?D~KhW_ouiq(70BdAf*qq6pe2wz0Y)Q zMI5y7(SUFbY{G6t2|hOQ(=x_S9bq>sE}Yv~uc<`?+$(5b4Di9>C$KvnjU(a*i7?Tq zVuvWTQM~P2j{f#_*3ZDrfUcjY{UZe=BzM}OLE)~CNa(LdPmGCZ+Nr<3W%AzW-@1-F z7)m@8+728*Q9N1`y0ksXBy6ntJc7Mk-N5D`t){n>do3mer_X+-D)`2lP`-<%zl_j^ zozu^}k#|+$){u&q>M&LgT>7#mHfqhiUy#0*=_6bLd-)}Jx&Ij0l-pyRxWJNk>&zff zj~9qfTTz)-1o$om`b=Z{Z2xlIW*6(I>E8$Yy z6&RUb?b}gE2NGQXhe>O8ChEg~-v1ck6~Zl=-Q=}?FEZryn7Fcu-qm()bMTGa*a>R* zRgCrV$mo{_Wd1+0~qwthSo znDkM&)p)be&xdWJd%`ISR$!Z`A-77&vulB39OA1-H8*hH;G!@c#dj{XOR)ga(px$u z-ykNc%J1}zFx?`PqXiwq^tvKe((;0xhod!1Omb5Nos|t_)HoV?I?K^6k_Ow+$V$~R;Cfq|knESi0aAKEMD^ugN6Wd>S5-+Y;8cv17fT{+F5aF;I5#nN5!vd=vi zeAyFWDrHHkscGy+UkIO<$2?h1oU+^Ww6nBeRjOTel#GZfqgCy@6H+k2Qjo+xtwUpU z7TNfY_#UQtANpH9q4s;q1ruyDdrs0}JfYav*DI@0#=w1wPWkcsI@G)(xRE&qN|SM~ zrZIZ!wD)c@^a6(pc4;)B7T6v#w1x9wEN#&E6N(XIMuUf{F9Gm^hy340Ct5YO2YC-9 z4CsNq+&6?ZV9|Xaeh{&%OL{899odle>EQsnljueV8zCgW10Woux|}UFh2opizUoHg znH7w14a9JZG-D5V7bo=L)0EzK*Bv1%jVNvU6LJ^7thl(s+-i`aborbqlfb0)GgjFb04RQ-3CPqEbcz(0H>i#VnA(jIN3Nc{MRA$9 zvnrUa)_7^=!kGtfhBDSrm?uOC((NhXV`{VW-Yj^Y$;!7TsO?Zb*Z3Z$Box=uSG z0yEQhPVC1kCT6AF0^x-jLU)k*F8{R3H=ER%u`xdh~I008{B@>Y@(=v*#k#GawxD zzlxLgYfcxSwxl1@!H_H0NyTZ%7T3bS;?kEhdNY&QU=$0mTfu$Q@Fnd{4%|V_essbM zyBIhKIxOs)7`cak4znT~lZ09d8Q z!+7DxK*V%GQ;}GdqjjL!tL2Y@%55|p{@w;KIiK=gx4LsSoExv>(~mLSup{r>k@YVI zpJu77Plp7ExN0eEW`1y$L|C_VJB8RM^M-V;?aMR~#_vq>b$*W2KrtF0RXa`|@j_x- zkSC;WtRu5c&lCG%6?*asOz6VvdH@U<-Xb4fb0jqW6?1g;DtBZgn}5=TACwmzXN)0l zoDAJb)p*^&)W_I>QY$AIKreWmxd~c1r0yiI-Ba|=@$xK7S*4`+<-edFcOH?Bn4jT5 z@Ko!_+sBXDMrT81b>WWp%?VCpYUeIqn*dxnlMRT=#C8}&q(z2gA}^a8IRU@PcMhve zgYmCfIh63VU_Z;sxlZbtL!2zZxY-+F;{k3Ka>9cERJ)v9McGqX=^?_Tm*O&lwa zJ^1>lTdbfGcmqBud32SS)})y9nCb<2bwWRWhC}6Eez1=|M#RICK;Ji~_@b)NbmgY- z)mdOHYX})&F=a;0#OWCtHUc~_^e1q|SwB6+ZcxwlAKo4bun^58u4#Rdz0B1UF!F3qp3scT_FflzfWo8F*-Of*PpA;hPN*-QLS}RG;%i zWbHX*{;53J=ox9%Gk~?-EPu_~^y?>1u4Wz4v-G@oiczuuig{&WTWrZCV#9+})h@&~+SeeJZFQ0#>HDz%8 z^u5qOUSc6p(X~3E=}^5d+hze2)N)X6f5Ovy;OLnfEikTHMLD-aQ~m z;VbgtJ<4qwvPP9(!ov4Rw)x6)AkjUS8pBvo_6ZTIIH1_Uk0ftjK6hlZInyl$qnfz7 z8<{PJaJ80k@*}Ob6sw9GX76z8k5a4yoc+{2lOM$ z8p~QVr8d6%xvG}F>?Xmr3A>|d^B3jkwAZdmcLV)7KY9#3Q%%n&N1yT*v27whLBk>o z`qt;*2U(E`wJDo^FnxIAAq3}orC$*!ebY+KGNMC~{X9$Y15x$0tu&^xX@$;+l>vU9 z6MkcSecp`7v`AmKO4c5WX%9HNWA)z4?Z#t|V>;dn$p6H+3Jf`;Tr(bf$vUsSIk;*P zbVfa?wE&WwXkpVET_67ZpJz)%(T5^q5Xm|e7Q|Bm7hKbi2@YsS1lo8KTj2&sZH(Rw zMxRo9fHf4&^ivNBlAx~DTH>-2_?p%ChKK2y6{;euG?wX#60uv7SyZgo3hZpwcwvwd zaFR@1bh`U?1$Q2X^YXK^(~tbEsN|Ph*`<#rH}9wY^X|8gSU4X?xYsJ^@E1KS-I{zqb?X1|-M<$x;%wcy8qh#c42BwUaRAhAKv!2073W7<} z=Whp)Pc1o4vK{c5CHlrr%P6NgJ|4}(p!HrA+ms;8r{<0m!Mt9Z>;<=LvZ?KVnNP}< zSG05)ruW{vqDv&cNmXTYX)a>WAQLfLAdGYp1FHD^g{+4xL}Qv+CRQSPT15M#3FCA?;?N~7hEP|oPD z6~x@;4Q3k~+fy)YwRMl2rYS?krZSM5Iu9n1q{RF>lIOJ-wi-QeNm?0A=ZLceJia;0BDmSMdr*HCK7$ zOSZ8tzZd^{Ki3pnZ}6di67jKX;p-<(4W~p+X}X9QoD07>tyxuw0}MM@kudcQuR339 zJ#v%Ubd;ZO*pY4)RKT70o@UDm%0v@V&6^qa+r!9Mx#-P^ojXqom$Net6qPtzcTbi4_N&soipka$ zJ`7fJp0O^`P#(t>$?vk48YRtMKkC->SsZzjT*!xto|*doy7?vAzm)B)Cs6y*652a| zVP!N1l7k&%MmTq8&wce2wm^iokBqeS=A_j~#~c`Kdpz=zi2(RPv>gvagti+PjKGr{ zr<>YMfw+A2%1>oL-%EeM^I8~nX9OK*@aqw~>nC9mu4fEUwQIBHe#>Re=XV3j*PL(& zTi1LzXAD$kORQwl+CM>9?F_x@M=W)r+}ZtLqJTKfZinV{d}P0|bdd4qGyuLZQ7K!s zDGKn2>ls;F(_n!uzL6*(cE_nnTLIxtMsFsyQ^3c&&&hlBp_D1;rr_Y~=l+neH6m4Z` zhx4TeM1F3+A-=>B&f|5dh)vWM?r#pL2*sbwuC{vjt(ue_5n8{~k`6}swLp6pk9XLb zittryUZWgeu3W4|BTn}G$iOp){ZY6pl3R`aq`sIEmvylcokSmQ zcJXZf=UVIR?hu-FE_Ap@NAh+M0uBnCU~rA0g3NrK1zqxkFf>U)TrZK{&vbpmUn{4N zg1qJl12>|f`qCbjFWBAjVk< z0nU4!2;kchcO*)2&t{J?CqUVwLAd|>QU11pj?Qo|QGbLS^XI;b-g0nCLEvYN8-E!Y zmYu#WNAjeYukB%5u4Bif-e-tfRnJ8q&w$F?{Sqq^va76TCHSUP7zOyw%ce=VUI}98 z?6_uC-TTE}!I|`G#&T}~zSo#Ez&CWyj9~vvQ8AD7J5H~t7wOg~6w8d{SXji5{JgKK zuXZFfB+JCxFDXh34-MT~V}Qw0*()sjwB^^=+u6q*VXN8sS#Cn-1D`l*F>T(A@3EiQ zWHlC?8*}axQfCL3?ZEJyqY8?xg71TV6689A?|Cj?i81%xX>qApRV3<_0n&P~(Fa#Qc7W)*zSr z;ZmG>q=N@p?})RT*pSp{H#;Gs(Dvk;2w9F@rrZmv;AOt8$T~@a#WpR&4<4VFBpk_u zG&E_8V*0ycu^&P?hD?I`7ZCFSbr?Z|Z&k|ecpw*iVLLt%-8WGKJAnt-h*3uCoBV50 z(@5=CN(%6`M~NP0KK)bGG;ayiH^bYAv*t6c5DcnhK5BVrn-LV<#GtUWM)31i`&Q~? z;ti!I@7JYCe{+1;ctek|UpboC%rqm*hj{3EH>z4HBa6aDR-Hv9_Fn7KHD;OrDa!ZG z#MtI4^&^9ks-{~B5EY4R3AHe@?f5qG`e}yvFovJ{PKfsWwcC+ox46-s{QjZKvWF|0 z=xOSz$=aOQiIJq3q>c)bstR(d`Doj(SB*jarU*DooKO(w4FtrqYi?p}$jA>#R)YeI z%>lsJ2dW}>ENt!_E(+Ysd2tnN58%O0TUR2$~DVX7>MNL6k-|7 z3}OG>h+ctvr*OfB6AS=r)ffLl$dP%-i}GIr^3EFmH#7~80Um|4wmjYbmKd_Ks{oMz zaQy|!Bm5Uc0(T$~fEgw+0NAIB+{KU@IgTVyP~?BXtw{fc5y3puc;LuMBIqoFe|Di0 z;(__6FaSj%;hQxOQL7)ye*|HFZqbClVZtdkK*D(_lQ-myK#()~`}(5ZKpze_=|%U&S2$f?~9PLo7NZV9{Af4E*VZ{uh{G@EZ)AWdQu+D-Dq>(ftJ_8~+zV z0E?QSfv;v6pwF$rfH~z?@?g6;A!r{PaJ?NNSo8xCcxw&=@UPV(6qMj!4Z+z#K;i#D zV6Ax~=w>JI+&loB;EWH>oW%ftnkRat0G^!}0{rWp_Wx_B|29(P`ui)#YXHFCjyeA# zP*6AC|Mkzo2mIqw1Bw2>&t=Q!H#Fu;1V&%P0Q~K<@n>tH0lz_`MGC+_1_rO8poIQp zpgeE+07=V8hZvOWt_u_wp zh|7$CzjGS?Y;8B`H*^L5|JMGNivJ0IN%=1b58hvX1Nd8B`zLsp{tqa#LJs&_wD~7^ zoB0n2k#K^WR`39SOW^*5FLVBYX#jsKjs66;^C93nz&{ET$Ujx|&*M-WMd12XRlxr) z!~e_Jl|ove0KZ+s1N=Lm`rkMkZUyJ`Q$S`{@Ziif2Iz%e$VV2i$^r%$cAW_NegNDz tB=kFu?I0f~hQJZ)7SOpP;LCM(z@I%eq`UcJyBq-vZ7{&gjQ(fe{{h$kSn~h? delta 46867 zcmY(pQ*fYd)GV3_C$??dwr$&**mx(lZQD*Jwr$(V#7-vX`~O|LcAdMv?5C=0J*&G{ zcN5rED_A@yv{Lpvx`}p_ zA=`Eu(d}@iRP!UAc+9`;S}-Qm?ndN4;aAP{ZhkYgmJqr3#~ZVN$6AG2C^EmA+=)X{ zo1btGo-|wBzR6f;=$SPe-a@3eKT2*-EP&(*bT+je* ziKFt144aQEWi}Ec8uVGZNpSxf*I3|&Qx{g3aBWmHV@y~~bu*Q-UCA>uvR7OM&X0<8 z#3SolVwgqe;JHmxsnTjk2H|pQpPOxzpSc(7-{^Tus*asE)E=^fkza!Qua81PTytdb z7?f9H!;NM2?Ig)WV88I%G^hAP4Re5iVOXBob!kn>)b5y}ZG)#VSk6v<78dd6tnjL> zCvLZ8<{}(r`5cCV!a9|`q&8>YlTwEjrL=s>p1rLHMwZDx$J|A zRCY0hRGmj2PN{wiYz`|?76qlHXzd1P-wkdoYt4;d$3i1SNOZhm=1_Y@qBKCbVwI|1 zNXiZZeJkEcwV!@V<2F?5c~2hOf!Y)IhRPH32ARXAIaqf!*Z?et>$odQ53MT>@#k+U z1Y+9*&Hk|V!MQIjU-vhhjiY<@pG;!5mXt~zDx=Htj)B>Wv+0tKp9jNOQ(9!so$d2NZtA4%_NzjjR3 z^&9;+yXu9$=JBIe4GY)DM=oYo=oaZytsAebE8|uP_et7V+a@^}*$n{AS zibl>Ao&_C{m3*&Zu8or^mC20P#Svv8)?ewX6=8_0AI^*-L3No_J2ZubCqrLGSavw< z_DJ0274jhlQwv&M<~zVxu%iJwYgfYM6Zh^mT@fq!aw)7ZfzQUBXT)y|xv1#O->t+o z@N-XN_Bo6TVLqIG{lj^Pa#MUdVQ2KxiV}+7V>oM!&FjCrn<4S`-LHz&o#2_GvG^-= znME{Q)31qnGt5E~8rbd9dl3ya-tkpdI1NI2BZW=~j%avR=C}dkZBZr?^#*2=kYpMT z`B&0^N|@RXGw-d1ysE>g5kh2rQYaJe66b!G!{LvV10KU2hVz3&jY-*8&jm><0+BsI zzpsXVdk=JM2u~!-~JC&ok!(p%_1LMIRQ>>(rn1iH@1#=6i zpw2mGfAGnchs**&p0U*k%k7SCvq^s;5+8&33ym712U-4@%W-`tvBZvfCOoPttiUlU zlf6$YS$$_U!puEP`8`@lif*pdcSnwGtp!o<5?4rWJdr|b{O4>(BHrt;sF*5tj#+{FXMDKoYO}@Nfrzo z4Fu%p&;KT_Fo*_0*>KT+XgLXhWCzuuO=^WC1iYqv_|o4DZbni2aG=3V!=dlicl0P; zLQ;e8RyZVd6BuQkFoVZu%r@Q!%8RP&I4rS{Emo*&v&UMFVo=q|@xQ<5HNcS)gP0fW zn{N%gMK%`NF(mF5OOmza)Bgn&Km0!1y{TZy!kFPaB zqCYe!FeC--*M`=x4$LJr9nMQ?4E$Cra+sy@;sAc=Q4uw)f6W4dkgh8nimBR~v1( zYOU<@y>29@=>7}(jcf3|;g~;~%#NO*e*QrH8j5$;!iNewWxbFDoUK8_zg%F?pR8Fh zORd2%?FU@dPKKp}y4+y{Fvudhk4F;JP3lugYyesMH3hB4qJn3}q{u##FwC^O@<;U| zJ$2VTq|tF!7+PTa&~T4zmB<`c7eW3dfP23nL=vv#yHhT7i%?y)?>+;c)hsN6I8Nu^ zOn%ky57CV&acPr_AA_NqdHGi8Vl^^%uHpORa0h~=7GI&?j~+2p}MFNwd&gfD=8I z7TlUhSwEJY6BZdm09i1GF!;zbQA}~jaxEWkRD2E>MKXsZt~Z&2h$^qb7LR>4TA6W( zyf2xxQ)cVm?PhuJVtemD=iALO_{TmDh}SZ>%h36pH?O-NH#c93zQ>)6UEoGZdT>TR z>mcW17&RMTE-T+6ZTv;qdn+^mU~Q&3|% z58M&z-%{TyoG<@xo=+QC0U#RD|0@E$P^33H5CnT={PaNYiTQ@!1MUETLSkvkV@cYf zJcARit0MTV)~0m*ry6M`wW@rFjW3!MjoclSqZpj3oGJ}GGh(ONq@40YNnhuN6B!qhFc$gs2!VG60rviQ9aM0)97{h z`gm~?8o1B)(A)SnwZl?`&%E%aY) zNw%}Oq_DcrcaNXX2SK=N?wT&blFJ04Y;-gipSif^0KPT8({%xmzR0t`ad;TJxlw}I z1`QYMbHyA+;+oSiyI#Z0E3klbw`JuUJo$rx&}GYv4H|gI7zvJQhuqabec3G5x~Qj?wGlvnzLZ6(-d1q zo+aShw2!lFVGe7wgQ&V>JsrWXYw4|Hog~k++Qy9G&I~d}gr}azQXMXa%OR#b7eCZC zU?29u2Jx9ibjpqo-(9c~g+}kHd^^@JTk##8~wy@O- zRes-#O`y_^SGjN<$CUBmY&VRy>tP%}55FzXyiOHPsCHS8S&05+F^eMBh40X{{N_jF zvqt3o`lSZc+oaBOPI1uDqp}yrh|IlSyR-0cZ8qS_yxZ4uOauk=f2Q|nu3SeWQihY3 zT^Bc*KkU5bD}-!b$Odaw4AtU0bxl#UZQelo#sg38lo9C%i<8~sxr^Xx$qFPOs&vty=72A=4to!&ciC*&S1>*#Tg}IXJ%Sh;$lbX2Cn=}1&ie2E&;QBY~O`}D9?ycr2|ge zN!Dw+^Tc1W-beN1-Oj<N5aNIW1y?8*=Q9H_&lNT;OlDg1-=?K5&So{xJg)wPi zeStQ_Q!oXj32h=iW6D7YT-{{7EeiV3Ez5E0j@@`DIxi)=f#2k1=5vX}+}06A(ZA3` zGsp)$#}kp4qbHDM3oAV&*e%y@x6(sS?*j8h-e|1H)0L8CM?GGiT%WHHUnsFP_Q3$j zxaUxIx!@=#l5DQic4J-z>!cYX#)w-uwBT+ruCQ_AUobbT{}2%GV4RD}R3l5tw{Ts4 zqS)xNOrXJ(-NTX6M{^G|>`M^N6fP{RBk$%$j=0LavmQ`ilF|9X%<8bzHv3BH$tDg2 zSE&2wj&ZCk)H+41+epjI%y+Z?7EK27QZ`fN#@_6X(0&=F?J6hz$S#VAmebYs-Ko6OjSLS3+s2J8j8xS^Q`MZ7!8))P-9#zf7 zLaEJ?sEeKyJSR8ZCGD3pUDe0{XYrWvm)?s-ggCfOhM@^mGy;C&$^U8kzx&z%S$Ntfqlp{$Qt ztl>8!N{vy&8v3VOY+3csQf)!0_&do*^on?%$SBo|8O-$ALm zJLN3r6n2-x10E7qS%l?zv(qVfHLbX3n(T2|6-|@OPj)OP&5A_D_0NwUlQVht=%i!( zkStyad(J{TTte4Ko3d>nQ)if`#QO6`u23?A2mjsMK}n(+GNJOHm1&w~1Ohb{O}TdE zxF1$?tvNY1rP;GuGl(N7((1zVlnMRHk_GYo{{qAoCLg__zZvFkf@ z=rUTB!7$o1)%{4oA7aP7QRnP_?PpVv38oRJE~9KAC4Rg$#LBjyf8OhihDe`~QoI2V zc9h%~k`TV18XRmgst;&K+4oZ#cg2#JlWjY;5{Ld9Wh@+^#y)S}l2-8=nKejUz6` zzI(7zT*_L$`Fb`tGvj1s7hP3zrum;!8*Md3>ElDXT}AB*1Y7C=ap4CZ8i&FYNKi1A zkjl&YWV&OWRGjYK2o5U=d>wWN+9t-ykHFFt9)%uQ%yAOaW#?`uVO5-E>m@t~E4$2r zKx};0(_aTbheu><|^nXm47tv?sQKatKU^Gc#U5zNqli*3AMzE%vZ7b+|VC&3nhE zu5;Z$=&E(&>K*;pbo*l&wnpE!o&)KydnNz{W=08iZNTG=nbU~7IXBabe>PWGGCB&tNh6ov_dp0`wCr~=^3BlpV46; z{VVQ(ave5)mGqVRLQOpO^p5pPT{TJb?uV~ZNtN2^Kt#74S3??f8B4{$;VaUFc!;Zo zWJvlelYq!%dEY-0nldlPmH=8v|CK&T_|NPBN;p32JyXbBmN|&KgjqHh7Xg}iwOwK0 zgN^)YmYj@FUNx%_z1)lW_Rg9@_c}Xeix8zgqf~%=mR<@+^-F{_t02ruJVnR$*IsNu zC>sY%mi%W&l3>=G3ntJiXMDNrJM;b&$_jy14=JdaG*kZyp&}5ZXrb-P16#w3i&N(4 zq4-dB%a7Srme93e)d?yKb;|+isOkuOz+Yjd56tX!TVL4_!QGs2#<*LYz+sp>GAnSS zuq|`JpPNvNl1({o2~TD9pXGSLt!IqJ=e&yZq)V8beJ73#!{XFU?g*j#HB;3M>7d*d z3}iE0Kn+4lez#KImp z{PyU?hUI}(+hE5r{e#wziTS7KcpVcp$SqH2)iuO<$s`UB+d|($x9Nd(hm5LHNzkjK zy~rl-5dN$hHu?5KHS&;eETLYs=(eOf&&&-Zauuv{rLb~kQF7$w4IfrP=EbjU0)wbJ z75^BUP7kM8^oCs?_px=jSv`OR|8F|ft zYx3M!1&YCBn2XCrs{ls9VQBtI`q{&fD|ELI(&L$!{*2&8 ze=F@}55-(4Olnzr6bz7@K?TAKRbFSOto@b5U}EBpokh98i^h|X{0b&_VPU7P7+#>SbsE!G$&V(m_5C%d z%vS!|$l^C6%idub2}>VW1l+6W6*e(o)@K(DxAdO1hBcjViIqJ*C`Z62%$-*pOlWHS zfMy#M$BHP>PqS}bvnde}Eu_8u$rYQ{Vo5hLMXS?BM7#&9(}whnbf)HqWI^B;(G8Q^ z1kpQ@o0r^M%Dp>O&QLhPG-V6YLa_uyNFcuw{Exd$nw!8$>cHBAr}i!$?1BUViGTwE zVg27gr0YLE8k;V-n%Lj?+6x=^JnmaVHJO_{(z@djHO&wzW~c}4PVuI^)<@}opruzA zBD(KvG^;h7rfQU=pxZ!8LO7CJ=N1H*a;z=O6ZZmc!CCnRZ!vGzFb&cJe!O_NlaICq zS4ax@%y{&CynOU*_kP}Z03f$NcNF%Bv#tAsxT;bCMk}r~A+%F=z8oPu%cXv`&T|X2 zrKWN!6?)7imZDT`cnwyD%4X(+EOJ&ApqcP&^;h93!oHJ*@$tGDp*vo^1#LZfF91&60$E=7{Fr;3P(r@3fn zc->Zj0SjyE`IOmxk0m$j7pT2xt?HfpXdc^jC7RV;O`>bnKP_>gBXEus{mmKfo=1j^ zR+14Yy3u@rf zTF#`5yQxBY)`whe23bazN9X=pjcQrvwZ>z>sDOwzg?W+LvRG2`&_W0aOD@Do>zBmc zQ&?H0!+J*GC0b=JCB7i5*+y&QTAl&&Q^a{x;~uT~z4Q7#2G+q({l@Kq?7rvt`GTE+ z>T|AzT5w3L;cS^?y>oojrVUv14DEtU{qIr201I(^GHK(}P`U6T6nvM%<+bpPd|goc2Kt9Ki`cdaKlercByPS%AyBE3<% zO^tjTnG)@pX>u219fb8?;sVMo#SW}MnSPr+I3_X6&z)3}Ln_BvNWLp7$YhTyg93vW zeGduiyu{V{uqvKs2OF>w;TVkd z7eaLS9%pkqA2mcBT(swLb}_OKC)>&Ip~j^V@)VvUN`z5{26- ze#d1|r>Xi%$@u3GO>43_hqjHF_HORPHa)L0g1nFYLYg5(CZ)Z##u1I^TSz3MtX#Dz zPp!>Qgi^P|5P$FosBPb6!68g9)I=#__yB@j5~+Mud6d7r%S`*M6P*HP?F4()k$mzO zVELy!VzaG?NaySSeq;hWh{1#iLxp;s9e*WYpN&wT?`0vGM40&^6@v|t0||ggCB5W_oQ?p?NC-`4?N(o_N9w+(5yadn{@%tM6fB_`E}hQIytvaZv`Y*kCdEQTAvF z_UIqq{RslHgRdx^5}WaeaISci7k&>QV5%{9;NNetRWEa-{rGdfVB5O&osWN=ABMo* z&xx?T$8=Q@`3iPC-QHY$B?1XPQ8&DQnbh_1hR60OIwwclvu&c4Uy9IKty9g!`^R!x zuE#0XDecBe`{pX9K*J|6!z;1~qA3&bKOz7BgWR0wad7XX93o+W*gd*qiA@LwO-5I0 z0R_`amJux?@JEEBGHAj*T80@q+(;~5Sfsx?BT7EWQv>2xY2`&VDpdfVkt}Od(V@#s zwFmJ1b%)vqhk^EZ5D{GlL%~E%TuT%j99^s>7?lq-OD;zp{!>ZDOn(G6p^SV=D%~7Q zk3$2)W05yxdFlXg-k`d&h>~SO;{ht zT#4ZmT}`}H^IN=gDb+t50$?5NPA=wdyN9SBuX)m*e2jQDg#-u#QiDogcLzT%dSGq_rwL-OMGm`gI=i`19 z)?Jp^1WR3`iVe`B*D;Ug8>FG%a)S4EsM741C=Byk!GUoC!=stzY3-v(9fg{A2yB7X zWfuc9CwD-A@qH|>7{b3OyKDU0!ri6HOO7^$N&!KWGddSw(xI}n-A&i9j>2&}ZF)us7# z6%g(gqEb}DH*CJ;EJi2;A7mbrh8i}e659+@jx=>W9<2)|C##@5!GRyz{Fkpd zkPew4(Tiu2suyK>2Ac4RjaHL%l`XRThbp`9gV&51r;|fRaQMz>YxEoB|A`U#e_Aq% z{ZWiJ7!Z&yED#Wy|J}R*QZiuuw;huS45+6utuch1$i*5YTqFEiI#y!5sDNESHS0pe ze043Gbr!=ysr(c5}#V@%9V>t((7oH=CD!qo}id zk5s=~Orj!d+GU4zx)d1d(k8cI{wgce^p|)B1#)`dQ84b%c{nKFX>X-c}H*l9lk;gglbNLGc=w;{Xqu4Qh@^1CeQ+f zL?tHN6l!!M#es2^TGSm|abak3UU&78JbZ-*%GpT{apT9P&Ts9QdAA2iG+nRBTYygg zA&o%LY^5h?p6Z>KuP`u-c}EUmsBw#qo^`8^ZaBEw{Tf>QR*Jc+a%=zgj_>ta%=cMq z_E?O$n-tx1GmMD#skGsorssb&e0~3B%IIHsQ1+S;nCEi`EBz^p-aAlte2r$c^KCHW&~W-1Ph>wyzR)=9EM37 z5;y`T5Li>|AJf^(C6hvh^258DZ1cN$mzFb*RdXcPo9a4kiD31!q3B6yHc=wPt7|~X zpWuT)a}iw97c9a^n2R{M-Pghq&%%mA)fV6`=Y{zR5_R>O)U zZq4)v2{{k?uYqUx-lofWQf8cE>_b%}|F3?NB>Qu3-{8+m{aA=4w$mJMz5I#(@brHB zg!2By%DeV)QCbr+GBKfutkPv{;$Q3S1d3R`+6h&bTsY9~>87UMc3L={n_KZ6wioQBt}yjsODw{3#jC+JbnjpENBByux5f z4nm!untK?x1n*?w$ym6Wdx9>S$TRV|MH#VvS+wHCN187ZTWIzB`E+_wP+3Y7 z)PK890Sk)eF$v@=4(I!({O|2K)9z8PtlA|Zh&Xq_)14Hg9r;3vTt5|3l=|Bfn3C@5 z=W~f-3*w-sCT+IxaTRqC(u+6t&B%DVR)69qIMGH~H07-*_8YEwt_7ffrpQYe*?$eY zRE$Y#&V-+dL=-)5YH*9Q_m)@c77M|z?&-jj0`1y^^6kol%3EWpZVfhzx166jxwMZA zgQL56Va?pqQzyz6v7x|T9?ZXz{oPkYr&!oz0!0b9KaShq%4InJqRQC#Vl2`4suJ>y zPUJPW5%`%Ww&sTDlnbHbsFvgVu;I(@^x!f6s)=P#pzWh;ogMW2TJI`S)+m`YF7sl+ z0N9_5IbcqYibF9EIlPsVPdn*F8+6+wHc_EXqa$$AJS+8Rz%TUT>r4)1wYJ7ynMUnR zLnvr6ygJT>UGOJT&v&TTHKap|${$!hK zGi+Ovl%-XdU<(_QmU7g!je=2=Qgkeivsh*d5hA>0)CebU4#%NrsD%qi&u;M8q3H&p zbk2Qaeq-p}xIsfd_NtHgs(;m-Q3Lk08GH>(&c3oD;#;CBp) zR1r9fd=L$p>0q+T;mQm9JV{_?yP6(FeFEP?4JA^Ok0KHCuV7M1k`YHD6kfz{j8;_n z>fx5FeTMDOgiEHR;>j1K5lU+>`h_*ONUgdEcoWh|Wxd*P`OLNGO|*HM%R>vE{jO;- zl{Rx{Cj6~vo;Q9L#qogocCUpDfU{Ct)^3~}F zhF=EmAt-(g<8+sanqu>)WTL6W)Sq-4?H=b7E);C(5$nUs!dQK~=q?r6>k5m~mG)SC zZZr4ZqY@4it&X<8fm7R>eBo5*+|qlM*jvopdTVN%I55x7QnEVNjmGa60IZYsRb*5g zjasUCSFfY9BhL1FVS=edlVXoKgr;f3>UP*hM@S=d=ik8S?Y~J=Is4w39{g*dXoIbR zrG&1Nh?>YVDaYO??j>u^3{Lo8F#2<61C6(lJN;@A6T+35+@5#XO)lA)ubORfrB&!$uMeri^y^Y~U5$|bdGB#g-Xbri&Cx2pO-$ZvuXN#<3^YRAp*6^y$ zO^Dh_r&_l?g358nF^;)uv=(jdBsg6l#^F4VMmzCUNS^0=b-d2F0h=q1zkgQWG3Nwq zULAY|F|hqg$T}6V#f-^eUMWY->lT0T4Z_{_vha;#-J>t};7?niDfPLw_7}~|^XR!R z*)OO5@IMS$WZWGM!PHa^vcU3mGG}1cgqr`*)l2EZy18O+^R`I;Y%H=}_`cca*!wm6 z0gjMy(s0DZZvf;Q14xq%7`i_^_eP1HMfYs}S`=h|6l8Z#dJgXNY-8Y3^ zx&?#3q4}nAC_{kE2K352&Mb0WeL{++{6GhBVfv z@BA^;uR4b)CWA;Y5o z?y2eaFLZe#%u#>J8NbEv@&=%zwwuAwj$5n4RFz%tKohR2yFS?H=tz37=VQcZ1yWRx z;#>*AKF?We0*5T`7Wi-?ipHeB`YE;|HMRoixq@gN5w2TrM)1FpZo=`q!rqv>xoD1 zq24)t(j$k-^pg2uH=m`lVcc>CO|R8Olx0esOJ`MifY2b>+0F)AIW-j9bvz9Xq-!CL zzUw&0X89We?UoR^Q7?W=jLPwglg?hKtATpFe$sBhr0pUiDEeafdD?U)woYFT4H7+jUL_Dch`(9(MA za%<2mU{+b8s*XN{{>3N&{Jr*;FR(_GJmxE@-5GS2J58h14^%U!#eBNJfuSr7VI88# z@tV>ue(uMT5uvK3VA7e5#?24iK@g|YTMyXWQGNKJ7vHYm*{S*5@IJ&q3e~yhZDE;G z?cNHA=`^DCC{jcC9XZtK^jsP< zFbKg+dzQ?*P-_)A@Ky~+{$op+%5%Bt5%FlV3i}Q`sz2g2UjJvb zOi>+Q$hJq(h_v6Y=i+)z2}hJKmN*$_=j}onX*x4z`O`&}&Y$>hGDH|E(ILExCjXsn zsL31u-lC zsH8R~r?w!q9yHb&Q?t=G9I(xShJcm`!J3B|zF~4f>FoKiCelqtO5bw#IPCbmY(I6r z@biD%Q09WyBLn&(&T@&;(sa_LWDVqilgV0eU?k@#Qb<)HmM9zq_uY{ub>yKlAF-x_ zjI&Il&woWMoL3inXxsj^@)oV;4zcDA_HgG>g2$*M;SQDyzHJ{bizI; z-8*7Fs`0R**Kj=n{6WdP1}p)Fl;b;SX@QD^0xALYVOKOn$-8r`KX=$2ciPmzwF@)P ztrxnX(w#WEA&Qk?t*(A*(tORJ#XiR(3wW&z;bPn+^`i9nn|px9coKWm&hK*=J^h@H z6R~kte4nWnn=rdU*_ar(`#F~JRHv&Vi`t`$M8$mG>-G+{ndPR`1fKnikih?1{h)^4 z+?3P97#oR)_mOE`30YN8`~Vg}w#}tNnPrWeAl%cdX|2V;NS&2F$bEj1UiSW)$$oK@ z+|kg4dE`ylilge?dtHIP`OioG?}ZyP?57gzMUGHeRn$-)p}csnGpWgDyd{0r8QQ4G zM->pzjeL96pc2d|NX=Tqp^RGiN8JJ@ic0?Z?eyJrPW5tg!Q>R_U(B5Vcr7#*wjV9~ z8TR9V(>TLeOBaQyS(jse;sCU2Mz`OdL zIl7_poq8&qeBGO}+8sOKZ3v&7SMP^98Xe7MHeR05s{3OF+Vp5mWkWG@nV&>^ZS#=N{T6c(x0$R$s}NhuYqX9MG03M{C=TL(t#G?h6IPp@4MH6o)oB_u$4^ zd*babJ1Qh%cW8`wp*P_$>_G=OItY94!($#lzufxS)Rblo@Rkfws`HiXL%kM+67E~0 z3$EOv@)hm>XOTkl4z(?W1{rVh7W4PhRA%a!gtOtWQun2ahe*LL+sml_u6i2$*A@8( z?Xq~`7!2L8{b!&6A^?>SMe0joe8PPrc2&Fah8Z>oT&3*P5vdz=9$A!;$DI)3Tlvp3 z<)`+bVKH% zd*laz%6Vea(ilEOD}IGT(R6vEl+$)&{P?qe zMpm>$nI1?DeI*ZJ9!VX%qH{=H3v!`wNT#@HcTR$;Lf-U`c+ZNmV$=QTDj+U?P$j*N zMClg0`Voeh1ZgBarZ&Cv*gO4;tLwHjGP7ktKjG|`e@LZQH3pc;I;IL@34 zIpo}XOPcnsu=tW;kK1ej_our3I-PyT1B+b+R6lTxX0P6lQDOA){RQ(NK3_mQvEbtN z)c6z`M3tpC#~K!c>I|LZOX+Yxdx;^ov3V>H>$s(Z?X~+(TBipb)m+APrM`iY41I5} z4KB9b+Y0LBJL@Isae^guc|p>T=VAvZI+;Vtun^=5b799#5hd~=J~2DD=I5~uJoI_P z#RE3~UU064uj3sAcNC8bAC<6-sr9=&k>^FnU96~0$Hj*bawKh;7ALGO+arskOHSV! z4{XY#%@?)0YolYWtM>xqqxwgdo7zw892*af%O|j@3Uab(o3sSFY;cmIF0_>w7t4lH zSZeXp7A)@~VQeL{;rPw)PJW|$X_%s``SFqI-N`#lm9b3XHQiEr#ly9h(Z!Xpw zu3}H{j|Lo{VVy0C#CHA1Lj>l)ujFiuLo5_T6D__-zXPY}@G@qa+`%7rpa}14yc2|_ zWz?4lCWZyYZcBec&OUiP!kD6&$bqd8zacNJgZjYvU{7$5M8&;ut_b94lsawYz8@R- zh4SBhSjV(Wt#F4&u!m3R*-$=V<(EvFzeFC%i`BQ5TXnnO6wA5pSyHP!gCaTqp?;03 zb)6f+^>3T0duloC4j!ccj0B^u9UFjVwa1-dP|@CxTxMr3#?BLj9VCn+IVHb2Ha=+i zj7+d4{p|e){eOAZ|HN4j=1+DE|J{I_B{6bR15ie4$N1mhE{FQDP$EQq%!+1aC>$tg z@WfJV;AC)M>b}wji9xt`fAPs|s#>(F^;gvAc`gQo)zwg#Kvx2AH*L~dbS!H+TWTXq zd%s`)GNc_L#mC=#>~Q|`@iWYQe-GdnV~gp%RiTYbwn%0xD#0e5UbAEKw~kGnbuySZ z1x(opZixK%i5b1T0;}bdEfUinxC}Ev88Mv`(rS-PtSM~u4oZmm#Ge(HJu}0~e7qoM zmpoI`ByQ~(2`635O4Twua0pLJ8hoSE;!eE`Xbw@29>lCCiZgcOCcOei&B)kwb?ql^WNV@v5)Fbk0J66=3Ni4* zmSFL}@z=M?>EdkF_@K$cJxK~rv8b#bP_K=TxSTP642dBQmf%g=z7P(1#LIl%oC!9! zRUVx#&e=~?{<0z|Kgfa!Zi2Cx9gSDr=tI@aJckyCqd|m{o8>^uk3@}}5l(ZlxSDZ~ zh>ON$1M+T_&%m%tTPR!#{NQ-4w@#_n(ukn%IM=nwjIn05TpkjY3kn=~4{R2XVH|Z; zzp*aG4okj&6)5|yjXDJjI*+Ej&36g&b|DnyE1Fu5+9V!3%+>>H({JL(_w%k4nUA54 zGWk<(=g^8nH-qczc?wcmFdooE)iz4th??`p1)RxS8wMaJ;3Osgd2?tluPmkY= zDq(Xu6sslk;ot{vcS3WaL-cY%AHU+!o5E!2q~FfBlBvs4Q;hSYM|6q4AL$KvE$AYH zLCSi+lnJ1LkZ2}6mC*<(w*AmbgSEoSK%_+ed%hw3@EVxlge4~TqvQW7m;|jB;`&D1 zseI~w&P-wVwesNun)TmczFrqu{WeR%=!kjvzu87CT$^04+o$T$ z?PaN@pa~^qkrrvp!~^k~lOl-^RHcbz7f#3(V9I|3Xt09|ztw{eiCNLl@FVFggQ(Xf zczLn8!k&g}sg_=ClDVm@*?J@n4f|U!N$KsaiDRst4MvpGGNd*Mhy}wMxg?d$&e@Zr zM-?+zLMmuXA>~RZNfC6kYgCIfXr`Ngsi{J!pO|Hdr7-)I+7y%JTZ5jplR_mMh>{!2 z3$j%MqM`X@)!3j_uTn-ee+Bqd5%RpT!s6IIEm{jhf*_RjFO z>z^X*R6H>F3uP+g5TYT}E_iWvN65|ld1~|7L5Nt;6td=cC7O)o8K}Hc%7@F}bBav? z>4JC-pXCWGKzMk*~POt6vth&R+G?g6u7I_AHeNzdmIz zR*ZE{`MWg1pW|nGk8|Y82$RFX|6(bLeHJ)K2FxHnkn-qD&fOG!rq6s3q5sl=(9Sm) zHU*03^CChez+h~+yhS`k+A%mXofO9as|QY8Tsa`eI`k_Q*(Ra*?=Dt-CZ*!fs&@gL zd)?{Kkda5l39W9$`!+FBE~}O|;&YCEb6p*$fdvZ6mHt*k8zX=Sa$hV&lB!fV_BDn| zN>+?@EnFmx>#sw!goyv3gZLqHLYls%JjTf9IeP_RiDm^LVkg3djq#>Fh67oT|D6%RMB zFFfDE60@R~kD*$PZ45{Xp84ssuu>~fw1M8~`i;pR%bZ~(RBCulmNQFlalT1HZ%H%2 zNpHT!{9Bz#v?%MUW&5k;SS|3JIm39r3%(+|@YxMzSn)1@n)XIv=9^IH*+)BW&wt63 z^G&=t;_E`48cobEA`*BPcrV4bmam8TM6E}no&T=;hgJ-hVRsj@FIbY>tb{F3A zK3iwL4?OBy=WE|x{nK_A-~9f8h+B>DNR}zSiZ1c+0A!WloZplc@d8ijGlZf6pN7oQ z%H3|Q#gYXaeEd75j1hkXkgxqJi|WmmG+!6h$E`RjVBZ;oQsmhfb zX1~FeYicAt$)X=wnP(CM4xTBisoMlPeI_fPfU6@qW;YZqr z-&W0l0X;&ms#joXV=v18i6GR?nfUSB{*=)j3;qL4^^335cGAWS zfjTLaU<^vb-AKg6^dv^vo}iuKV`Sn5v#{SK5hkhD2gdz*Sm2cW7ZC& zma2bd5`)X6+x6D3+mH+RoYo~1+90KUfuA7MwBN=iV`osNHx_?>UdP?K z2@rA3$1$GeUj-`Mube#MiYdCr_XNFTqWvYR$+v>)fQ|H+`+W^FIBX>uk9*CP{kVw4 z1IZR$#?1DkDB{ty8f}f~obpkXgdHVQD<-&-jveI>UiugI7aps~w8DL2sK0?RFMbQw zG|+bu_+bqK86=YiBUBY%4|%MLmJ?-XN6m9OPyPt2bX3UgO05q`WsoVHCfb9LO`lJDM&rnV;`^(h z>i^^FD`P5&maWmj2X}XO_rcxW-CYh22X{Z%;0}ZP;O_43&M>&!@bT_TzU027`$uQ@ zk4|T&YVEbER@K`P#1-1HxGvN>uSbDa_7Q8!xrF_gj)9%pZ7f#LinolH#i-3;gd7d) ztP{M}B!Z{3fFr)fnlhCkAuay6%%e$w;mU(b(#+8)G^pseN0-W?qM2;p(dzp5+jQG_ zom~BdnXVJbOt#2&zf1j@`>TuMBDqHYwF{1EZ^(x`8_2Sz1=dQxO!+vY=CJ|2j>oqq zk4zF%7a&bNL$Qdp63m*_tV$DBsSB05^{jS9k&HUB2UHsl~z=2Pi;+J6S42G3r6I9ZodZZg+9Y9U;^W!y#YKhW>P{ z0B0*4U9lH-{=yw*kPlh=YH&0zfXJa}=ZAtPI8c$bt-WrldTtAEt7y(6?$RRek{s`j z6RH14(c24{!lLyE_F_+j;`%Rg6Eh{q3~eg_xcXiN)3yXU?jM3}8IY~pZFUE3&$5bo zBWM>T%(q@Q)Y;YFw5|22!3Hv3#e-R&LwylL+ z6;P7Do*97}dydHmtp9AVss@Uh9obZJFi}AV#=29oW8~i01^Ggu?itFQMqC}^s^~Vj z@H-tX04rukzAdwKpt!VV{AVexBD|e#PDX~I+Bsd#FTh)Nzs|37!%R{v-i0=hrGwJX zDV~yLgg|HXhY-~WlH6w$x%jZ!Y*;6qA3!a-?N`f8?a_KCBM zqwz4iE$M&z`YTPUzh|}--F_z;9=cr>9yC}lQDJZHrxrR^$Ekq*4jiZ~Xttw4E(ex@ zT@b`71^ph7{P_w$ixzHL;py54&Wu9`=k3x@1i+4K;0HWHAKu7at+=%1Uz-4Z>(B_bhZCH1TXh4B}0MeOnvw zs4^0?s{!+Ly?l&iEjgMoLj`taoEP_mI+MJFAs4%h*p>j)rSgAlx=AdZOF#~c3r3XB zYLqnxX_Mb(BRL#|^#vGnCPIV1qc0^1*5joC%!QL5JfC1B%?Exk!LN{kE$Dl`*o$jX z*Y6|&9s{|LbVDt@(pr5oc||)Sg;r!qAG!a)?58*~b0cJY0&Q@+egH$G<<2l((LZqU z+Hib7MS(+`n6~$YY&ZV2&A_Xo9@=JP$v_{OpI%I-Z@%M$^rHu75Qa*sJI_60Md_SA z5=B7kd%8JrypT4gvAMIX$YZZ|k|zwvMm3)d zg)4eNSjdmD8%(6xQy?VK2XvtM*bW6++p7!-wpDu*NPcWSt_7M~Cfqs=5w(}3H`4ap z{(4&*;}pLJaiZ4K1p`jXs9F z7EimCqhJmu5E19C_oE{WsEqtbQp6yIVaeCzFl)}~+eMZyCGZiu{uK?WI2|>|D9G6` z?AoyC;0z#s_8aWXq1Ok-=H1|j4@|p1a-Gl*`A?KYp&uW2Nocohqh- z)6Jb$GG)r@W&-^u;7d7QGZVKuy7T{5F-}!x*bl5!1OP${594 zF;$#&?mhsR<{O#?#GLt8|;OqdC z2X$mkFKtqjrgEgf*Sz)#6*4K;Jf_eiymf38wVH z-N}4_ZwL}z1mH4nukP*oJMwN3T7(>W(0Fxp-DKItk3&V5qn0*^>>vRebb^x>@=(w~ z2)d~u{F$~i&;5_dR%xCVt5EZF$6O2wWRf+BSQyuq7#w6|6;vUE{WbxPBECkpq8Cem zVNR)HeX5Co0EmT${ci{?Wc)tpLm+rsvyBv?Z{?pSCH6dX)F<+P`W9HH@ESAZvLWC? zZG^!Zz`?*YL4t}hz^$d$rPc?{3rbz$ea$U0e>Er-DAHBY7-S&~sxcI51J42TqUTx+ zL{Z{*@P9OCKUq_T;rk+Tcz<89MRWn5{b7vJiAPw1!^i{}GEb{{w_l!ji{!8ID?oYW zTqcF|+D%FqpL$}7SZ$#$$>>o0AFD;N{GO8-ni2yUUs-Gx`6roF1Ovz>isbg;vD>#($ zYuiE_D;3LA8fj?3101P7%%}`g}bV8po)flNP855OT9pm7GQ!HA25(%`a~pGF<4QpJm>p6#kB&v^@!St}Wk%?8ju zafv?Jp+kour3zS~Qe=kqG#Zy1Nws|p4{($!9+XmiQ5Tc6h07b+STx9G!a-z zvZ}?X^jzb?$?DUx$RJjNKZ_X&amQ`7lR(o0xG`9SD9KG0-xzX@ToActgWG0*S zCWR+=Y6r5mg1DY1ZUDchWny3?)eFnj*3)sf@3g^o&dUjK+Y7x{XEV-O+o=yX166+{ z4z{Ed5uZ()4!*%4WoUu!FD#ihFg+((S`o+Mr;}vDhfwFx$x^9ADEwaeHN)tbYlpdx5?!j4ZiC0aW zowS!1mTvT6Px5D?ZsuL)(3=nz!xRv6`Ex-uF)a_Df%d{3$a1WHYN^2tKK+W*wVq$9kY9B3 zF&pebVa4-yP><$he$MJt`g>&(B%sO!WNy7Jp|@1&vwyE%^TiI}8tv3=xl6yF=c62+ z1zp+&&R;ukgsae<@tVy~;!;Y9c@^OVQJNcuaAYPUvDC4MV=C1u!ctkP_9lyVUc(6uEYYdiL0e3O=8j}2f2dYSxv z%Be@BZstiBqJT%+li)Y1JU4K_V{SlG?<^(}?Kqgs$^!ixFjMbA2bW<-p2i-3%p}dK z`t(obB%|ht-4{V3{!&MUmsx`Hti2$VY_-3*K8{lFt|#5)YqicQ6C@qO`w)iV2}>cn zRlEQyKH(8&oRpMsqj6h|p80E_T~a_uI%c;26M&h^P~}Dsi9O@UU`Adjf8P@?AVHW# z67OH6u+1}j3#NPfAeS{0r{nQ_n%QCKQe`1abPsG%4;{}k$tS$epAI*R30Ce{L1tfS z_#;0qOU&$5z6nlC-Ju5bJ9tZR`yR;z+RpgsaJozD=PWOM%+~l22bBRJ{;J8pkARVL zZOa(^vCNj}9=qoV3S$hX1~SUK6SJzC44vrIYJAN-tg*purqIDEe)bNZEp4|)hpD%) zfKUF;Aprtn0qvrZ?@c`}zyIkHKeF!kVLrX+T+_4UEx=o6_Z9?`Yq)14U;ZK5D#PC@JC_xE5Gx=g&{K4W=@8R!cU;*#6Nxm-u0h;~CHH}l zNha#23O86`@-DWk*&4T%j=B zH_@eybBudb3K({Bcqn~8NC@K8Bn8&kywEZhpch~@ob%h#*Vk)w>vJjvGycM&dj-c$qDrxzLV3 zi5;j4+u&+kos4UD6Lft2t}3o}^v%As+n!HgHzRPbP%;8z>eY3T_kzED8zlYEBpFo> z=eLKRRyN6|SdUao-yIv|*TVn(pWJovMnt{)Wh7Mx2L{ISpCY9$En=Xanwl2IKQ0*_ zQUdWGi`weZn*EIR*p-yhdYHB}P_i_UU8`a|R8j-HXJb)6oE`&em2#$6Bf0^9<;&O~ zDZg8k3VD?&GG%L`U+M%hQ1e0)m_OwpC1QGU`l(sU|Si=*REhl8L-A^ zYViBu&&95MV9@NYH_^cCDLQGI47F=pFvg5)GsAcU)ohR{D;VYp<$JSbXFPZ24Hc90 zO`3rD95Q1dB}IvLnrQlL%#<`~b+59sOuk@!Mh1RXI}1C%pcpF`5x>;YA>~Ch)|j;= zU9z+NfV>2>uA-jh^qjnv;lv>zXc%>XBU=Srsocr1*LPjFv;_#5XL;}nu#owE(o_=5 zl%pfV$X>e^uAiP_NbPV4^^Ka5A5&&+-BmK7YF(GBawBQbXL?S{(mn=ZWY*1!6J(i1 z>R#+#woK>DqK>#yM|L17ned32?maRpUfRU)^nwIhnQg`lGREYxw7gTC6VEU<&av>d z_pxXEWkoo^W&^y_UtT$rZd7~0+;Xqr%`|K;rY0=JiyeUIpqozMZLj7{Lky5=A!roH zjQ&#xohcUykYxP&$&w^`F6UdA$BbNs^U6a{K952tBNc@4_DZ)NQ|0o0{d39bZ37QQ zyA8pJa%oEy#zWQ0YWcJ##qv{Q8v@pU_u#w_Lm=Vkoq*I2UqmS~O-$Wp#Xd_eiuDHH zNmZKM9Yc6UKg8LlMgmiiAo7>@+l>Pzmn$8?Sx1qg0qoxmct0*DzTs#?lT9_>N;Plt z#;Og7h++GUh7PTT_O+b)CX4m<-^m;PN%h(4Nh6rFni^Fm2H)=EqdjkOgP%0~0xJ&< zh9X8Qo`JB$jXtgx!Ts<;DmS!3YB#*y$~TnVsyCe7CYS1dF;`Hp#2;I5f2*q9m&G+C zp%-QGu0ThZ8FCxTw!hHrf!Oxsl4rbiaoJYzc6L%NSTaey+l*>Q9jUNJY-UI1O3B=; zQH1ag%)h!FDpZ9EIiKg<0&upkv?VYR=$ivTyui_3fcvs%+2XpJML3Y{={k&V}FKlL&TQ2o9Y{dfWFpBMwMw@K6r5 zMu8q8-l3VWQhjqPK!kly?0LDhQ@RjXl?n;4*V&nRI7l3d=&T{5Yc#m`mZe{S0*K6Kq|HFSzp0@7r$;3B5$*ye$HK5v!i#F?zr`^v z5bF_+1qYAD;2m7Ev?#>uy^cZWJOg7yK|m9yS-sxvua&nNv${^Ns=yi(quG%)rBedf zSN(+mU2f;zkTCowu5XV0VnRNsmb)edfDw{-Gn}c;_)b3p!j!kY5%8wu5iT|y3ZS5Q zb$TB7Jle5WcB9EW8YL?Nj_Gg213KI;$y(uEnfG9;Yx?*$is^YhWWFf)yLXy)U?l(e z8s}f~hKT!hue8sETuVees@6N4>f*HZ1A>odT@%OWm!WDvgU^OpGMXXb^r z6F)W5gufG)Zr|1gWMi{kJC|^b`J`}zV7P1b1y@|D9Z?IzL8D!6u5L{|_Ul8--JGL` zKBy7JvDD@T-yg_4lQw?4!JfOBLjUmOyd2Cy=eK^k*7}bVQ$XWR`)tQ>>C{>4HkO5- z^x=`guODAq7W8Le?L^wQE20kVJ2s|E`EUNqpojkBjy!cnub>{u0h(d!TISzc+_p;} zIX(d2&!y$8if~kOobZ`$l+RQzyoOQhQ8%?+$$`3X0w-Yh$4srPmH~rHrHeh`GfnU( zX&@AtADIMdfbVW&|4HZ!f)VUH3(g5Ozhar-IQ#9wD>929@flggiJ8G*2oLJum{ZU@ z+nUSZSUj1eN+z3*yedZ++_>!?$+&!yRcmnNK3a%C?a;%Ow7(#68cHyN%e%fnoSkT= z=oadB^$ZY@)c!*TW{u3o@f-T^026DFln*U^Nh8av?s@4l|;TZRmuAW>f66`xTny8&9m>EA5jdIO=5{hICfnoL! z^!AeIY6^%ER|{O}4o7-?` z$thhqh8+!Sw<*$7T03;1dGJM%i0Ej~!0%>DKwx;WkyK#%4&CpT$gqn$^y%h3fJg|X zAI3W%d5m~}DMIWSc82(9Pal@};N?KTPjrx4>=}dp+Q#xTHkAJwqmpN!@~-pM9yZVH zfth7M0<}?KhV=y)8!3m@a&RXfpxZCd-J3rn?S++2>4$0rzc&Ga_vBgj4W^wx{9DJ{ zn`Fc{A4$*`(N`1HX9B*LTQu%cB3OAywXRlLa75*SpuPI*1imzx9hr8mkS_lhV!oMC z#YBj*qWmSkz$s}B4SDJJOZ=?{hc|26a;@Imq{~BV+d3dHzY=Xu>{lS@Hau4gNlgs_ zB5kb#TY+&S*L_CjqaAGpoKfy|Wi2~%d)jxuT;{yZ#X+<;oM{K7^sRX$J{01kyg^)L}f2oXQpWeK_fDRWygP5(sow}0# zXEC*$sBZkNu12YT<0#Blo8Meh0^DOISvS=_yo8x3XBS?wkGpCb48;E2C$)|Ek}Ir^ z8->r|tL~lHpX2ojOMe@}cE{uZ*={Y94vYbHBzk-Bp$rncbAR(hv`i%0{1;0IXa>T& zz8(=^>T`uCc*ENFKWgT85UzRXh;*rdx%1SG-Ll%h7}xH3{WSc3ms@nIl7w2(Yj$z= zCybgp%WWwQlXNQLV~78eutr!ijwia z7Cf#Iyg94m>};tDkNgzc)2LFg>ARv!O5I9b=MVCH@*ee%C;DW>giU(vDYpZV`}fD& zG3$n(j_B8`oUA{39Ry${|Jz32Oq;XcM0fPJ#=xp%|wd1YE(w27!dq+oI-scVRTp2 zRwsH#QcI_@9Lul!;_-w&(E!y`IWnvDso9S}8viZM*1xp;+zakO{Wu(eTf4EjOKSX{ zJpCkdi^V7WAl~4!s%b9Ht!E%KCH}&olWrcb=KAKJh<*UqGeEirJN=$6BMng9y@QPz zZp!)Lsr}%`qtX#H#)44cOb>Of!?7&?)UVK>P2}*{Pv)6Y{Hx!^1Lu&w@!!SU4 zo@OKc8V#Wx3zBIUN<1bla*FJpM{}~sgNFe+w=a0_kMYnyH*`I^rDV^*n zY9@uCI46y@-%)gDPk2;NjmT{quDWQ%W0gRtRwr_PqLi%Q` zE$dcx7ma}BwO2RmKaHDpeI8=jTeDmdt4DXmK&XU$R&hcYcVpJLtkF!)dnH%)wE3Ph z$=6EUKCb99kHSqEmz^NrI&`(wu!#Ds%PY!IL7te7Ym6abn>HZJk2edR(1A2LtJ7f? z0h#H@yiMFWZWbu)$*ujzM^RzvW9In<#{Z)l+soB_9qzP0tuG`Af z`aeDx)=$m99F`c?t@cY2!>dQ#=Y(G+9bp$A_J=!(aZ zDs+J5SVrmrINN|Ld?A>@&Rz=)Tt;CS2u_&;X#;nyW~;i6IRh=@lYb~NUzL^MQ`qd9 zi-JKq4@5uFfY&^1c_d_d4YouZ5o3^^kHhGVt0Fx)CnOIyo08zwg|_8lkgQyZQAjoK zR29UEYr60s^9 z&7Y&DiUt;9SC+hDgV_q|aZ|e7sSb#^Ao_9}#7T-8l8g);09JE759LFwvg{#NKSY7F z`4L$^GWsoBzkZOGDqC0dX_-KPJpf?yKbQNS$P)c2)?0m)tk=+GmfcN0Ro~2g0r4%-JZOgOKol)aoYidaNn2-k- z+b>E42LVKAF9zJw!q-6g+%gA+Ss+lLp%h>yE`}D1JIL7;(3C6=ha)RM+t7P7>hz1Q z#&zH$z6w8I@^(KpcZ=T;2>L$?&lfGXWhGMC7t@X$9tKS2B<9>d^zFjyfcyMC-kG(W19DSV>Q|WCH6IhVpcsGBLVa7ch$^aoO=-RWUfuR5by!{@p-pF0 zsOoDm~kpxqEPO;p$2JCa4mnt6Akdiy9REGubrNP8i zvh7fC>LEz3&Bgq(DB-{tt}mNRm6wH(Q%inq8UO-Hw3pN9DCHU*uvJ?PIQ(BJB-&*( zGSLcos5~XBc+Gy_49CQa&~Y|QGEd96hg4&q)Vce4*R(WDdz<*s-W6Z#FRLsHCObbw zJ|~ZdDl=%PSBL8jnvHvFGUG%a9KUyH#k)or?_0U=IgSMCk1cK1l=3yYFh`Mo8a1!f z>;sRdL(|FA8Z6gToB8=>^a~qbbC282JL65l|ABlgI-^)|4{*>$!ev|HgvW;ksBHtx9-8&87f(a}I_PUDX)j zOkGJ2eON)qigE?Ib9C`tLIR)!HOO<+0VQD#+MbEQ%kT!Bz#oDQ5&mvpkil79Y+L8cT_UUwu+jvu@_$n%}4 zyB?LNHKKf?$OGdgHutIJteEVmMg+T^jdOP-H>GD6;b{pRlCDT)D=f9{;5C zqp#~w;9wIfy{ZH5-%k$J8c15pnU$qQ<|0)oN@?NKX}=?cI)M@e$@Divyn&(|aZvuC zP^M`5R&if&(#;}_&V$$AOV>^aSMxja$2Cmowa8ld-n%Kdh5BxH@~i%65>^E{h_ z{SW%TPkR2K@>%4@MdxD(B*;pn55zBkU1nzJMXFJRten`C zWwKh@FDv$tfqP|jb~XLQhUNPyfgDX=#W)<>f?jevVC7Tzem{HmGCZx??=k7f#eO;1 zvo&=W!{*jxlBp!4;YrN9#`b2Cf}TvH=}8Lq(oC)<^YK{_dE1Z#XBN>y<9adS;4!|e zowu6Yue7u@6rB0Sc1{LTyt`21W)8I+?xwzNT)2Np`nu>PWRv19E{5!7L);=f>8f$K zON&G6z%|6fYBCRRJ704&^b#xkM8}c&2HW@r&Wm4^ZZbMj^^^+DoYtS z0T%jG8#OI_j$>YGqNYP6g+B^nEftvPZ2?ukCDr&Vp75^Wn9IWk;xi}^2W?XpJCcJh zDsRwj^^)wA>?KApkd;a|$KMma4sss6JF<*0K>2EAjtm{osTFdGEor-B66$ORl{;_7 zETR>7XwHFnDl?>+nWbS=LbwzZp0k(Ml!(%ca@RCZeF_EEA)AwR=A5NWHU6Lx{~nfBW?o{dxOkn(KOL;JHj1V}E0IhFrk6L`l`Iww3G}bdHFqT?DLl;piF0%>1gf}+loccjnzuqW1>JNx0n!`x` zl}hj6@`%Wog^FhBScJwkDEV)K^#@)y&NiH3T>K$oh z+4HoJ=mO;PfpFmqIK&YlZ+&Qf=#7D#b&@m8xlJg=d-S#VJN5HhTK+SO@6%Y`Q`E6k z@{9xQZCg|&&H8MZVion(6b?n%0kBEZdtWCv@@|l%-z)w-Yt)7Q)9Meka6>-=4w$_i z`eWjY;X3?INsfQhKR!m{uxOtOdn z(DC;yCAgloC<6%I!L9+G{lM&Fy=%|`oC*c_g84w`-!F^Y5r^{Wc~)+uHK2_yFH-%& zL5t1ccefNJR7+;h!Jlc*c%kW#F9Jax3xa2RoBL47&Q(mK6+R>%Ts=M{BQE{mZBtll zR>VcAKGyhV59`2?WR$B2Z9#l&O^SwN)a7+csbY}jK5606ewA5jUoP}oBNstFnV~sr zIPPv@nf8a`Gs%NbG9E2}2MBxS%r_+40Sd@!->Yo{1lxdrs{qRBEc{xIm?ab;7}q7FYl3 z2&hAKf9XHFZ{HHxl|tlO+wq)9j#@v#$vA0e=1#I<6g)JV8->KBkOn10^(zoqFQ|?Q zPe!{cTohMut>9`G0hi?-hUEtg@ak7&NtMiKMSBu#c00(md?cAgS><4RNFDJop}0o< zKA9FTEw@OAv|E*niR@={s*6WhfB@3*^qAGCj`Rr~UAGp%|0N8^QJT$5zsA{|e%&-6 z@H7h0^&AXRSgZ(qKMRV)r**R~0!YG#80dEl6q@W@dAotClO!NRL`5xZY+K9zood?8 zz^*_JsITS7D+bu7H+RbQyBE1x=$A}D{?<|p%|JOGPAhC`P-bw|h{`(|iUt$yT*!1N zEq410D-(es`$v_4ne&To`H+%TojKd%_h){_GTvqHoqbz%I&>bV*-aX&lS;CQ2r^>GT0+vghqP~M$aO2MFmPUskccN z7^gygI&3sErF_|kM%V7TJws$zE6c=w%KKsD_2HcS#iD7M6_tK>JI?;eyQA-6`S|{R z_kbS=J5R=k;#fpUR)}tbsHimt#>Lu?d#So&OIS!;vu zTD_(V5iB?|4_BPlrgN|~|7c60MJeO|0=@ke#q{s`mg{XDh=Mm-;Z-vuYS02Erwxy3v}kyt z53V$AUpOxqT4oAnA-tyiMVr&AwfjV4^-v5{eB!-daeX>rQ!2!(fvQ_E$$z$?$0zWk zA$dzXrDamdWO)S<5{GNL1nZ9!0oVD^BiCR$ixC-OuH+?aq)m0P74Qf*3_Gbq)+*F$ zGRjfkxj&2vyisw|#IRS58Y|KKv^Je2JgPmuQ<+=cL9bc`(|;r?`-^Pftj9=oGk`=G z>Fr2Bhr(lEuZ%74qf6MVw&<>6iOv1BfEP8^@Ks>s{$edZ0!LQnMEEE0Rm0sBRa;iw zN|lQqJ0|xAMZ=~k2sn(>lG$ud6GrlYb=63YX#1zro&`~JI!@ivU>fhulU@fRckZ4K zZM{Pm{e)n>-u0eiy%PaRp+|BqAjjl7TYr!GS31nzr)f{Br>Ea^q1YFz0kl~I0a$qL3%nuW*{|T?wA4Cf-1YvbN?Q|S?1%borM!>-;+A8jz zIq5MAs!HHliQ2AJTkH< z{?fu|zslO!zhZ`ys}-}AtBJY21?YPkHmG(U4OpG(ptPWjG4lBv|NQhDK278dLhuiS zl!C%YR#ZROF>!QrRKpDY7QTwD>a*y5sq2nsQM6QXbm2haaW6+LsVzzPVU9DwlTA0^ zNw)8d!~c$*5=i4H&sar2+# zQ~c@MvM6RYzf+HoxaQ03KjC{j{M&$ZNrRNZX=pDlT%ux)h!>4rx*tELrmX4?@GoFP z_o?lKI}T9{RHF6H$W6#(d)$7tR@egMXdX)3HB_KlVI7-2iG~#&;tUTe-Heq52`$wv zg>6!5`k@lqw|zQh0EaQ{YM#{;j%yLqI+n1Anek+X0N+@uhz>Fi#)k4SQ@VIx=-6Tj zt>8jhE-!sB&kGp!%vuMmG_ih4eA9VpPm*Bfz(hcdslP|M(QUmFPhY_2wUq@9xBlAK z#ENO1VsR@ZGS3hidt{%?>*PQEHgZKbAU+aB1TYNa5e`S*^7#KOO7F$|Xx;t`$)Q7i92c&*Lulh== z0b;j@v9Xj=UD7qLmYJUDmlE6dzfHPPYj*)_*-6`zq!>_-0rjUEZxsLF2)`jIs_Yww zmi=B8?59N0NM@5Mw54E)A4r=5@b!tnu#H2LEV>A!b&nQt`}%on>;42lnu0A8qW zNMZ=mBZ!+P;b1S!rY#mw1q!q9Q1sy>LriJ8uvCMZs6LV{hE(cGVi~x-!-Pb4$Z~8V ziseE7&gFLsfp6b0?rRPMqmbBsPTnVYJY8=2&g2UEHUs~?og;wZ;;@A$@r&e9;-Vf> z=%QdZscQKQwk0)L;gnQtaKyMg7&YNscSFye-+CMqYQ4qKrKr7N4C zdJ^HptuhRFZ8ub&Ra9%*`sh~To^e(hYE8**vpgs)Ii7mNccvpQI5V;9=iPb&w+1-1 z*56$hJaGp(D+}4P_rv2XyQ(&RFS*7s>~)s>YVN3TxOSPKacxCR()j4M0G_WPcI;tf z9cjvcJ1VvL^Qf!fz~SzycMXkuju+{HxAmPAGg>Da&<`G-eajX3d1~qqrn_;7bhY+6A?tQx#96MR{zOVO&0eD;IAmM?jdrlHBSgmlL}&!FTM2)_^{2kx-e(Zojsji=f3o>BJ$v8QR5jiBC-{E{3V#32 zUe&J@c>5JBB>#gzeHJKzC29&z3qMf&&;45Az9ZS-=PT>V7Q7THj)NDdW1_&(s=`bE z^4X!-s2ggoY*xF`dPZ(@rbUAp{0R)DA#QBZP*;UTgSncV?)GMP>U?{7i%0-c~Sa@A7MR7Y&?XWPy8T?am+8~xA%^a(!c3dL)6>=gxT_LK0#9o zTzJ`9BV!Wt^k4GNQ~f3NglYdHSL0W(T&m24I~mYg#I-D!0~s*s5YZvVFUmzte#CX; zB^7kP8NSR8&cnYAq_BW&>zV1Lp;l@+#uSrs|lrYWyQIucBpc zae#Y!5qJ3?^i<;6c)>(S(sIv3S-517`(Gomjm*VP1@*NWZdLyqd0W6xlq){pm?%~{ z!)m#nb>RBsr=o?)_d6p_=wEd=9lU|6uLDqVrr7p?MfGRI_zfMe?a-gybo{LDCC z!0YwPV_EROz!x(JISmVRxj+Ei(mwM<*TDGf;<_k43oMy6jp;A1c34OaMu=Y0RiU#{ zu9O%IBgI_^axa~h*G1vpe8No9P*EAgTx?Ny7Q7q*4!8;YgCmOfxtgRfK$AX~>goJy zFzD#o>3Z5qe}8?#HUhUE8AHKCVJ;9sAdT&yq!5M8&!go-&hlRE7fS|4fFE;HWQZWj zwo->87>9io@}{DTS73vGM>1}1 zNOFzGK_y=BN_W?>p;q2+yxRfY7|gpeaLf&-a%hHW*U^FY-}%~Y?)lOjDJVK+Q<_0E z_iGL~nMT;BLi$U z@SBQKc&nTl`XB*WP1G}m2pVkz;nEz%UoxASFheTXro$%yns5hSF_oI%c*CeAK-xip z*QbHAaO3+$>ir)$F@P!-ZOzY?uK72`3dzDHVNaM!w^u+%=>>3-`bog}GPS*cpAbTL z`NS&E0^^oEkN9*gXl<4n<&^CpbXjK-O##V%zA9@;L8dT8YBD49uNq@ckH2Z)99p@) zS;j#4$x@1S&FIw+N0ojqbG#f)GR}P5_}-<5wLRMmhYF~r*zQ|<0EAm4YfFg~;k^=f zx>;Q@h;|#f2-*aNW>(c_wC~J0?41-?K!7rK^0QfsX zoy|(aCEBD#sMT!ib?T+vP4dI8`ea2gDnY{wZz95p{8e0}He}hfFdyZv&CLmX7w40E02B-9{O{GXlXBHA$ zbW$U~Mwgbgy;a%K6npF{voc2)%Mg;5y!UH|arj1}`{+wdrB2Gn?L>i387svY?|q?!eFp1>FK%>>tOL3@SUQJn2uiTlv6stU~LQ4&-+e3l5dR88N~x{{keA)>-j z{r8|soTJTCYmMoy>6~9%ZR`L6&r(O}+T;mqgTRT2@z${RWav#5n-jX%^)S&wFonHg zRE*Fe9QXlh6m8+~LR5^1EJ>2EEGZO3Eoez)v>Gzo(~Qb`(C|>=>0To?X+1usDKW5O zD1ZSKMw#AFs>lmc3w_2Fqdvgh1ar4$^$NmsC>Oh({!#Gj;f?I}Qdj@v%`35r1;`-~ zJ>RA64srW>%NxYZ>gjLt>?Ykc;1top1&44SwteXB5(q~pc>WVzydH1saO0urd%p3Qscx;v;FAaW+$nFQ zZUZ)f#1N5s{gI10{NKK8bnVt$6QQ?Lb?Rd_lzsGlK6nYgKX_?^65V^}28IPFNxAIZ z%^sOQX;1auSkSXfOF;#25+X)PyCWgg%&MV`2mkQRI$!`G{fEG+aX520Adpz9*$SXC zh|V9etzE>kVqlL*AS+hKLzqI@7Q&_oBdN5&Gi}z0*~DYCi?5I3U{545wFmrny|@^! zW={1yt$4R2u|{fIE&M)n)r6=JxUNVY!4vheFE>8oL{qV|&vZi=KJ>Io8<6k$maxhk zFf4O}(2d=*PkoaWqXkp&8whH+{ub><>S2uPIH(YMF4JGPGPOpY@mWWxo z0@##Mh<`#^Adhm1|M-DE6oEsXIcXeR%T)D~bBl2stMDey>3-phy$Bn`4geEQx!G)) z1eY65qq^YzeeynSwk`|=e*C|(-U2F)Cj0-!o!~yWySoKsD&y@_2$Po_ z8k;OWlxmf){3tSP4kdU{%KOBmk%FV3yD1zVW+3|MML@!S;j@NF$f&bJdArT1j{jU4 zXs5Dq;LL-Cxilhz{h~(dI9{mLUFuXIeuH{7+e|~34Hy*^BQINv7tV2oNdhynt^?-o z+%M~W#IvtKPHSaI8k4u3Y+tvcbIa(3!tHsj%#WedLiI~cxKWe2BwI}%8!NB@c~-Yo7yTJOZ1 z2iH8`;Q*)s4$L@1;sv~OkQ?@qzM_XNK^8^d7<%vGOS*b*)}|uDLWJMqe2iXBm0N-^ zqQgcKd=ZXFgOlMJOBxaAIL5vbO!I-?(uG-)`N21`2w?QvDZh4o1&?Z3htvut9Fhe( ztJfUCM>cJyzy#jdZz$QOMQeHU%Sqr~synnspcMZWFQ*DOG*;w{GXtCHPg?CBgn8EL z;HnapX{^kC(Z2_V-)icwzJLvQ6zl~S2EURpfJWwVfi+rxCgK9mwHIp`9Vy1~{Cn8V z?Y|q#C*m1>iX>O&%G3+qI&TqR{Gi|{IFrB2dKp8?g~6Tt0R5si=%&z@v{*CKdJ{FZt25yT99Yq3qFU57^LQ*F74?mh^kX(bG6U{tKC z=eo)vAD5EAT~94ZTV5=4Aja%Z!0A|xVQJglv$&fpYI)7L@TUC{7o^>?R%BV7eciAA z5oS(ZQwB(EOCskdVTfYyNU_7pn>X6XViK}q#@?ks=zFhFFLxw z$_=v&0fNJxBsH{KWgXXbs|-<#4+J#7_r0~M**=*Fd9QLdZgR;e9e!D&6x&nzws-p! z!V4Cp-n1tIX4c8`YNZR@<}BHU@0lN19qeBa)0jR20Q%O2N3dsz6j`S_)vHCiMeVTjpw;vag`AWr=+gBSUIw zF;1VzR#y-SOAy16hD|jf5@kWa2=yj8SQ*PrARsXX{A!c^1}7m@cg`V8ZI#c7qN$ke z#7pG29$bp4RSHJPBU3*uDrC!4)=I1>R0>l#U`@EmV~x7BGB zf+W%&9s7DrW!85Kg6QAEuRvm^BS4>Q>*5TVOag3k?c^&N!pEVW&Q%^bt@9I|JZfVT ze1`-ly%df2c;#2+4Tp>yryKCfI)b#nW?_+>bjqcf&GI%)U)@9Cj)>{VX{6&lC7R}} zfu&<%{WtWcq;ioa8F|9uBV8t;KQtk}(>aP8NZ6_!y#x(oHnB|QkOjgWGYd-XV);&z zJ5703ES7Lm4Rp1#vE;tz?}}VZ;KKk8u{{UT-!O!iMmolIY~|1!Bu)-JgkB$j zc~qY1IZvXpXrb$PHt}RMCpRo9x0;;cHBk7CRT5vvXDWb@hS6;x+aPbpRE>6Gm|qX! zoMZ!qb`^O_!5TCr;ZQTzCU!!#z!V4NVT9ECwpT{natlEt!LEurQzZYN+~*h7g?^q&gl2zpHkr~93+i@tP2JIAVsa4cW$nD$9b^wCJ%HEPc0 zu$vQh=%6?blWT6V)!^CPcEp@8Vj8ev5#|&@q{2*RTB2{>79X`+!KLtkV3Tt_efS zPzKnGfCe5uEZdAPiP)|x@$p8W+@^o|tVT<6ocF+Rre4XJ11)F*sW9wY!3kjPLJFUq zFC9C7e`7uTDrI20oF$4R;XpD{GlS)6(#-@A#A^RJ!%RjMQ{QxtHfe+e_@SPB`-)pf z#}bQpS&!1z#FL^?z3mZQrK?bH`-^(apHm7$l-iwU_tfX=X9gRtN^z@qdfGRYN74#m zy7a%iSqC+t6oM*n&K8ZW|uM6%(58fX9bq=miK z8Z7kWnT?9*V4|ClE$E9Lf}-a$@G%rsQQkW2c`aNz)=pWZNe5^zS*f6+Xhc3;l(?L4 z1RG~VhTCjz;PqgObKmc?)NA3 zl05y0bFaB>C3{&gnD_zg6eMjwcCE9ll?mEgAmMO-h#S5VR>vitiX!1u#MR%^7al+j zHy@~5MG$e8~!3|3VAv9wjQNm89R7Iy!c?k* zln#(x+idm#@_sozj9>11kK7w8JL3YYN6deNR9{gHuV-q=FjL_!1obVp=XYVc7jdgC zyg;6@+z>)Ifvy~I$OEOydsOu+wnzA>=u2}A@mlm`TYT~0Bq`;$hv4N;xY9zAyF z$*Bd6Lww@B83ONTGejWP(m%g)Fw0$l-pI=eK-7sQa$r?)AeV9LX$ zNA}e{v^^@OXb~@O!{lKREuc4*!A}%mjbL$@;@7uXUtN5`og`yOEFv4?C3Z(cj zLJ$+V9>;)rxa1Z~F~Pv>KhI*u%OBaSepj%VO0iu}yaapgXOb>Qv1lEKL$8zkTl|8p zMqS^2BN$^;2|Oju6^=aAH2>bBttsK3y)?5kJmB|-Eu!u6wh`F>qqH>GJgO_(GlW#!f^;X4q{29!W}CnQP*@O$hdbE+hX9;~9>ZJRqw?G^Nk(57Va z^1IeTRX~Zn6nnNjSXQ;(iNaw_W?@W_z+KH|fA>QdBS^~wdJOe+TBQDKx@i%-xONT~8!#G0;>l|h zrRsOPgii86pDBL3qdtnpx8I1XSVOMaJ7PDk)+^`t(LGSY!nt>*pGYc^Sl|^F$@FOC z^a-et@U|~xckqii>2nRlvfL@*w&A47(qjha5Z_S3UcPHz0ivJavYv1&8`$rwnu{z| z3KJ^9{fXzDVr98(#TlBa5%3E#RdlFw3w;zO4oNvHyf;*cP!!V=TiZ9t8)aG@{csO@QxJD7@KuUC2CacTrQp!9egH{+Z)_w&P^Mu)Jvoh z&B6!7Rlw`y%+lMKqB?t6QxmPA+@kFD2PI8y{g@Vmj9E zH~fer>v)D91oCR5EWb?H(DKuLF;s-st%aiD6r+b^XjBwi&9SgPvgH3oe6U2sW`{T~ zO_C%HGNzZz;=ALprt|jsW%JF~!M!;2fF6&OgP; zXCf&~E)iDVZFR}sr{VBd?~Ar^lScK@bF2>q{YF+*Ab^GeFOWq*0Q*qHAYvhG(ssZW z;t2Mvi2kt<>Ln83H!UR%EHU(e#t|Bm#`g&5SI8J@$<*iw1r(7KrKMWf`eClyJX9Y} zep;rl+NX4H1Fy)m{55?p;k~bSmwICGSfB8i{CGZ)i&s#Qbh=O7-1yD?`r7F8`}s@| z!e?C@y{|3!oud0R1MPSTo5n6#oiQP&nnBzyVTi$9Te%E7a0ngPztM1B8wC;a^u5Ky zYh`!8!O${8*TsO+s;$%dxINh7AT3utYHa{#Zn3H@N#H?9BC-Qc!gZej*yD!V8n>Qx znAIXCbJa_pX7WJSIHgg&M=NXI)1>5-B&I@YWv@JKc$Mi9e%YW=vintP)V8$gb4S8s zS!oO&KdfU7&}|9(nGF9fFNKvG|Lob4_UisL z4hi#0Ry=g4%PBj)RZlIgR6p_*tya%KyUP(xUW65EJZ7;=50N!^I-k%` z?em29dG%KFur1{?I@#t>C~H0)Ti7WNFdyA3&b-an19K3sq;b9-VObhV@nSOgCTbQI zC=9T{73-?HV8y7dn`S&89vcty4)8$Saz$FKVPpojO@x`^E48l17FtStpB{-B=vd1e zTWZ%T3;-6pIK1~bl}DM0_!1(LMouLqVV#Q2d@bw|4Nu(ct=$-Tt+q*3k)3>5i`l3ndmA=2pFIjn5%@r|4#7TC<-~V8&gOA zO&%Mo=twUIxpp!cPfz{9<18;S@{5y5oxU)tH^HPC+xr8klIkjQ)bKVse5Al5_< ztxJHz94$I}PskRl-zKU>cLjS=`?X?YA&Qq7YNr;vmb#YqwF0^ZTAMCBd;)6J$u7JucsVoW;IN)$ z&cC@`@E_*6o%475ou{`>bvgsSGx;zB@dTaaxy!ZO2Kz0JxM667j<}_$hWg*R5bW~w zgJ=!e6*G1@lNj5>GibsU4D)n3m{h{Y5Ej50j8dXLP;b!|E5)XgoPZE*EsJbvK4*@I(3h~d=Bbq7b1X=*`z}mMg=1&aiPfo0TCvUnH8qO( zw`rI z;l~{i&3ea-x{G?u0?+|vB$z91H z7lbvt8JFs@X4+h?kMli43jIK5{P~+OX7lp|5vDwUfu|Rlt<`CVLSi~Q#zFY;CT6VR zH_MIa1=J5jwkIbowVyw>)&kEP;DUe)@!z?FD&xJ`G;x|goC#~$LAluC(!Xl%y^&f+)Q`*nN)c5+IUY+ZjJA)}iC)1k&T4o+X(Q_GD`H}#8y z7Ms1`=%kTkW-(Nus<~HybHYZlbNYLpSt#Lr7efPrk0nKvl|W{+K`7vwI8W};M|2eU zy%?C<UwSpY%}nqBDqOV^c5U z6p|&Ht@+F+zH;n3Vn?8f{V!u}HUx?hS&erSMP2fEedJkko3f@(I`8F%5pmh^W=&<* z`d)JKvnIJf7_*yOUzo~MC%iQ{X*gV;w#%#tg?Tu21=c{2pfj$9&~Bu26YRxK&~c&O z?OD@UqNkxSp{6#f-hARaKUX6L40+5T+$z+2YHhU<2(gvk^?6{hHm6D}8uwyjlnZ>3 zT3(BwdBjSb?Roz1{5*HWSlI#0g(*@piZn!xIL{muJo8B=d?fTMb&ktKa-FI#g;7^y zWEu9dOqfV369V`RQhvA@zb-dbhw|css#?zq(8GWkzU*lNId6WK&ZeG~x>t@T z%UvIHXCEMmm=_prJI*R4hdcY8VOZ!i%4#hTX?1fpSfn^4=S18_%Uu^xxOmsz;CiIiL6r877g#d-)55h?2wDm{T^%rU*&@=O+K*bQ{HR^6Bbwoe@v=A?t`Y- zwe}zovMy$o2ThmFEz(1K>siw0UTs0hQZC(J(8dZ%qfhWGO%jGfID)S8chzw6(0&w( zhHdf8`*Lu$21@ASqUql5-7Tiac-L*vbqe)XDlN)mUYODFT$<2af-vlEnot$R^>)4mi*Wy>6VdlKb`&lpxUciQ^}D zwtmHS0fhtArH>PNl&b6|2v>p8iR_>n$0~&}im=+t%UUGv3$Vv{jQ!S=K8=&)5C)zws8oVs7~%Mi>Ug)d&(_wzUIc7lM;<^z2ql7;@$mmoo-JyAJ}gZzV=9`z=RDo zW|2H{7#p%JZ7*kR=ybErtVK>Xfj&`dbe^Am<9(q8B%+A)Eb>d=72_KTCHrEhn`;Gl zR}otY&y{x_&x@0P*h%3vkNB|XAfR~ssZ+L%98;NOTUIHPt62H?&{k{OHLzuQ40oAj za~vga(YmBKKtlBll%IF|lQKBE(I$k}gM#8YNcwtl&d$(Ln+PcYsew&c5lEQm8h%lotw_75_(1R$ZWHm1eG5nbVeloiUl5Y=ASvML%3}Z z8pIB}>_EOX07!%)SsS;;TJIs%8GVpf4!=w>Gm0V=6Lpnq1b1eARz}7@L8pePm=P1| zzpZmUwfxQY`=Q`)5&P#8|HODPz89Knwy!Zys&GP%*)SF$$OESDrypDA=jz~#Gz`@J zA-3c?gx0M9lwuh8AH+cJQ9OUrVmqPKKo#CWKw|kz-b=7!tnjBKVf>{M-63B0k-nTe zue}^-zKE1Sw5X}G!BE1wt+lu)s`PCfLAnwfUdsvOWawaT+_fPlXUNNwf?aFIpOVaomYz2y*4}k(Il=f2ciNA0B#GtN zD_zz{Gq_ZuGb*M zNLE^=7DP|-Y#Kvt(?6HiB#}_e`*G2?%*hnvEU-*#l>Z122C$&B1HFhU%I~g5ST!5{joiQaI4!K;svPykNT@ z9CNy)$}ab_Vs8vkkrX+Bvn^Q!WfG zp407F{GOySe@2CyYimR5A|%CmapZGjt94%-`P zJtAWsHcyXxGuqtKFi`@_^3b@?fyDCrGtLm3%(Uw9cZ@+Tn&}FC6HO^zmeZfF71gYj z<(HlByC04~a+2Q(POE&P?FgueynhlAz{+n;2wlm=QXLv79rP+ls^aZ@g&C4N9Hhhk z6^b?k>}@Xb&iwEdaSH49b4ODs)sv&82;I#syO&{4)s5nHb#5kBj+&J)&beXqEur_s z_Y=5HV@CJ2Av%FlOSJrc7>;jIt$?NW&AmYw$43*i1D;gU9w7=3R1E(>EJ&{&3?};u zMX^VArlKj|FBu|+>U*{Gj#cCaG=TefLTaHyphy2a1G@sfbO=k2g`WDbE7E7mNc{K| z1E8b$s4uttI$T+A6~jGc#_qjENTXR>WbaqDcBgjZZPyF4K%=(-85cyaLL|nN<)x%@ z3)0Y}?qn=SVC|MrxWdQiKsZX74Y{h6336t~MhpppS5(mRGcu0$B4*ZLs)59XT#|(6PE$>=#4woq@6jQe@r9FJ3-rACBSd zb;PK5W*5S-ON@DP13#N%-=ovIMqvjQNz%KR;+k2Z&X9kPMBL*)rP&mT?2{~V9NbrX zN&fsu;&wtgcmi}gprZVRVu*oNhZz9QbtT(L(@3DLa2d|jmpvQZYRSU-)#aQT7MPk3?b!+7Zt|{l>BONE`V`i8_zvo z8nby_1ow{6x2?6*Wie4s0GLOH)OtPm!2TBRW=F3!^|?U#08K`;M}^?J-4zO$00bJh zJnZ|t#1gi#DUkF|MZGxc-(dz8{!+78vcHSZ>sCB|Oz4-jsxxw#Omd;1-V^ryXt9%v zzo`(G%jCcH9bN$AchoUFMF8=55Hg&|{2p*oviwV8hCk*ZW{`?m@Zh1N4r^iGTmlZ? zgsnU+52t^|px5==nB2J9kwazO95AZ)0>#f4meFsze|lB-!cg-KuJe&mCzZIb>?Rsz zAS$|gkn@Vnl@Z@Fn^&{qLtqSHJFds=Tda$x1u2cP4w*RV^N|JUjaa!mwrGptsGi_P zi+ghjb8}DOvkS4ve0_c7s^*u;7lhx+D$o4N>z>sZZ_vL#yEsw5#n9HMIs)kf40 z(%3}qO1C_--hl4kAQuVG=kP7ITV0XDJ>fU*J5(x?+<>onyfS!B9}W-N(#tFST{}Cw zb{{BonmfJ=U!i;IV7FN7QYT&WB9A0f4-G1SItpdU?C{}NQ)9}VYr5*ze-M|sO;7N?SNqu|koUDNp7y*dKqw%!djz;P!Uvt-m8@JC zO)mUMRThXcWrKb6V1=$id|aFbRGf#?I$eOQDs~oLxya?9tzo#_{yjfw{)WX&G@LW{ zI|-EgxK}MWhVEJBBqArgVtb3^JZXJ;G^o7Ee$LYNamvIt@QBV+7{24;>OhmChY89dAQ-)JfmU)_5T6^M`f*w$xXLlJ4BsL)ljS&W8_9=fD58n)u&Szl}`i z$gkjOUo+4~8!Pa{2%tj{XvsR>popN#5=3)Mv){m^Fio`zL7zI^9|~b3K}5B16q?Hf zVi7d=!j-U7`cR{P;iXiwROua+H4h_yb*BUjy#LUBo39Fdgw1!e^jx-xfa@ObJRE-( zISk0ZclG@3b8*%U!51kiH4_YrlOId*Da@A2MQgwm+XQHJmE$c`x2vJ1DJaM~zdhHc z9fEh2?A=+t%gG=@Bt?u{v>c^ld&zI0arH>(4_POD3C@%rJ$pK)#SwD0TvA)X)h-V6|~M zw^Oz@4-jbYGCP^M!sRu{a)Rt@)WgFWu=n&*5d2e)Gk4)kd{Wi<>&uCnRpw0Nu?W1e z^Z01#^DMt`D*KZqhvgQQFN;%&bagwwa5f=l;~;_0?lyXo-JzTG=J4vIcGt_LTwmE} z-@c#-ynI0pyA?A&JzB+Lo9*Gvj=@(p8Gpz5)NNpFrVr#smyI3rX=SnHVp9ZmQ@;YX z@`|)_cLqCL#}qw6oG!hD_tYf~k{e#!y2fsHa5%f2^iWnbef}^c%Rry3fgnR z2OfbGI5Pe>37cEONomBO7Li{}cBTE~p)ed4Yq0;tM%V=>|j@quY#fGIyw8k`vJZjFML@sTsT;J{7n-8z8HetPpL5{>{&SZ$bc?FT z569KZ>lB}eeHo}Rrj9d%>tVqXW|zUgzGXd0^|C%3ZFVERB{*|iUE@xjRN%{s9Nhr9 z1#EC!IR_Cn6oU?s9-w<|P-QMTF~5q<$_a~(#V<|AK;w{l3tD@NzO&_v4M8GVeh|nR zJB|r9MkW0Y&ri$M8I2 zsXX>)s@g20GG8UAnu|{jobKk+O$557JX0J}YV=ZtBquK^ujDUBiH*=IS@hcZbMaqA z9G=JhLhRRBpI$nXlpIjHuEknzVX-pn8(JKrkFxzu(P4O=l+@w!aNXUX2p+@qK8V~| zPeoA!R7%4+36PYJ%}9ZxEtly)PXyGWy~2WoDu~S?ZlaA3q`Oa-W&adJg9*D53i7WZ z3KYo^QU&X1v9f~lt4M*)tX z?zg73e~ZgPzbMQ^ivGiM0G>o3dhv3c+-XqUH*W&89i09LP9SOg>@F2_gIbIXdnkj! zdpwf9hzA+-+uH0NT`EvSJ$|mK$ zf@pS}JqDUjen*-v(gQ#&gIA*5-_&^$_n@Yo*d37hp3M{Gv@c;+)mv=Zv2id;1>-U| zKQX3h9Cn|0EW#CY(`E`=@VlWViSQ%7g*^cPddSp-RpJH1dG1FEqQ=W@Z|mWa`S-Gy zOm5D)vX1Y1{R}0zR=)TV_?0E2`t4V78#ExjW3Ks4*&Z9==oSd(>&o1%m8QkCuvuKDa!mCrJ^0Op``!Z}3vPA~@ zL!659(hQ-CG8>v)$RTxtU!5cb=ML?8ZaiT~_Ojm|V`CW__ERpHqAFg7 z+gfgM>0>c@etvv)q^SeA`^$8=AG!}+yWN{gnu?gvYC`Fl!+N5(k^ePJ09`E+f_}7- z{iCMUNZf-5k2Ax`1p#)M!6U##ben=5$Pi#(83d?j7yvRKW<*rYxla)V@6ZBe3~NAU zfli0nA-g~%BYcp@U?pZekmCp%fPznnkp~<}!-@m}!3<{qvjHGMNvt@a+7SkTHc9zF zG5DJW@Q3n01h|w7_XqNaM+9Mx;sKUi&AZURp6mDE*aq;K{nr)*|9>Dk$Zr%KaDsSX z-waj@hL^L}?;KZWfh&X%*2!{Vu1(z~F=;Jz2 zD=gqPa6!Ov5`f?utq>4A2;)VEfS~vb?PCQ)RG^J~pOdqn0091-MGGt!!}nL6Eag8C=_D#>aDoo-SDvQ7z#`TE11SN2 zCB*m(de!*@ebXTZB~GFP{v#LvzyC4g$3Nhx0U_uF4E`n7{a1_d#(zMjDN!hY8<4}4 z3gjB-+msL#k}Zhbo&Z#5LkMD-MhE;ykP@tg^H&3@_Fz!>9|%-FO$epy0xAdufVQT^ zAkRVMGeUsBB;)?7sqFp-{?Z2e3(E5aLzG~tAUwcddOUwY(SBfv4lK<>4ftQx zkpFO8KrqA&N(W!fzXceOAs~eQLx4;4!G8{iKF0|7Yw`78P;~eo$ajt#@W1P1|HHkH z`U9QMF@e{t|Gz~)|9iN`#6OVUJR{(*g%$s423)EKfePkT0RIk92j3~Ye*v9oV1ORP zn1&7l$L9n79q9Z2Gsf9Wkn@5(;IA0czgn8k{?pRGLQ3I4$cyNJzoPQ~hV%XprUCpF z`t%pnTlnV+i3>46Tl46Ezan7%0trg~fYeJYfd2{(fS~yc(kTZ+6d=kmT)@9ysK7nr z-_ZN7po{Mmp#3E_sGweOk>#IT0Hn1{2vs@&a#^;7@)-iHEL%V+j)2~;umk?PhXr>b T|Fz|gfZ|pd;O/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' cli could be found in your PATH. + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." @@ -161,7 +161,7 @@ save () { } APP_ARGS=$(save "$@") -# Collect all arguments for the java cli, following the shell quoting and substitution rules +# Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong diff --git a/gradlew.bat b/gradlew.bat index e95643d6..0f8d5937 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome From a98d7c6e1b4b61d3bcd55742639af5cc0ba891c2 Mon Sep 17 00:00:00 2001 From: Jerome Revillard Date: Tue, 10 Aug 2021 15:36:57 +0200 Subject: [PATCH 04/10] Tests --- .gitignore | 2 + Dockerfile | 8 +- build.gradle | 5 + docker/docker-compose.yml | 21 + .../devshawn/kafka/gitops/StateManager.java | 102 +- .../config/SchemaRegistryConfigLoader.java | 45 +- .../gitops/domain/state/SchemaDetails.java | 71 +- .../domain/state/settings/SettingsFiles.java | 2 + .../domain/state/settings/SettingsSchema.java | 3 + .../settings/SettingsSchemasDefaults.java | 16 + .../gitops/enums/SchemaCompatibility.java | 11 + .../kafka/gitops/manager/ApplyManager.java | 15 +- .../kafka/gitops/manager/PlanManager.java | 65 +- .../kafka/gitops/service/ParserService.java | 4 + .../gitops/service/SchemaRegistryService.java | 164 +++- .../devshawn/kafka/gitops/util/LogUtil.java | 8 +- .../devshawn/kafka/gitops/util/StateUtil.java | 12 +- .../gitops/ApplyCommandIntegrationSpec.groovy | 83 +- .../gitops/PlanCommandIntegrationSpec.groovy | 225 ++++- .../devshawn/kafka/gitops/TestUtils.groovy | 111 ++- .../plans/application-service-plan.json | 65 +- .../resources/plans/application-service.yaml | 5 + .../custom-application-id-streams-plan.json | 285 +++--- .../plans/custom-application-id-streams.yaml | 5 + .../custom-group-id-application-plan.json | 65 +- .../plans/custom-group-id-application.yaml | 5 + .../plans/custom-group-id-connect-plan.json | 205 ++-- .../plans/custom-group-id-connect.yaml | 5 + .../plans/custom-service-acls-plan.json | 25 +- .../resources/plans/custom-service-acls.yaml | 6 + .../plans/custom-storage-topic-plan.json | 205 ++-- .../resources/plans/custom-storage-topic.yaml | 6 + .../plans/custom-storage-topics-plan.json | 205 ++-- .../plans/custom-storage-topics.yaml | 5 + .../plans/custom-user-acls-plan.json | 25 +- .../resources/plans/custom-user-acls.yaml | 6 + .../default-replication-multiple-plan.json | 65 +- .../plans/default-replication-multiple.yaml | 4 +- .../plans/default-replication-plan.json | 25 +- .../resources/plans/default-replication.yaml | 4 +- .../describe-topic-acl-disabled-plan.json | 565 +++++------ .../plans/describe-topic-acl-disabled.yaml | 5 +- .../describe-topic-acl-enabled-plan.json | 913 +++++++++--------- .../plans/describe-topic-acl-enabled.yaml | 5 +- .../plans/invalid-custom-user-acls-1.yaml | 6 + src/test/resources/plans/invalid-plan.json | 2 +- .../plans/invalid-storage-topics.yaml | 6 + .../invalid-topic-remove-partitions.yaml | 6 + src/test/resources/plans/invalid-topic.yaml | 6 + .../plans/kafka-connect-service-plan.json | 205 ++-- .../plans/kafka-connect-service.yaml | 6 + .../plans/kafka-streams-service-plan.json | 305 +++--- .../plans/kafka-streams-service.yaml | 6 + src/test/resources/plans/multi-file-plan.json | 85 +- src/test/resources/plans/multi-file.yaml | 4 + .../no-changes-include-unchanged-plan.json | 129 +-- src/test/resources/plans/no-changes-plan.json | 5 +- src/test/resources/plans/no-changes.yaml | 6 + .../schema_registry/add-with-reference.yaml | 15 + .../invalid-both-file-and-schema-output.txt | 3 + .../invalid-both-file-and-schema.yaml | 12 + .../invalid-compatibility-output.txt | 3 + .../invalid-compatibility.yaml | 11 + .../invalid-missing-compatibility-output.txt | 3 + .../invalid-missing-compatibility.yaml | 10 + ...invalid-missing-file-and-schema-output.txt | 3 + .../invalid-missing-file-and-schema.yaml | 10 + .../invalid-missing-type-output.txt | 3 + .../schema_registry/invalid-missing-type.yaml | 10 + .../invalid-modify-compatibility-output.txt | 3 + .../invalid-modify-compatibility.yaml | 11 + .../invalid-modify-not-compatible-output.txt | 3 + .../invalid-modify-not-compatible.yaml | 11 + .../invalid-modify-type-output.txt | 3 + .../schema_registry/invalid-modify-type.yaml | 24 + .../invalid-reference-output.txt | 1 + .../schema_registry/invalid-reference.yaml | 15 + .../schema_registry/invalid-type-output.txt | 3 + .../plans/schema_registry/invalid-type.yaml | 10 + .../invalid-unrecognized-property-output.txt | 3 + .../invalid-unrecognized-property.yaml | 12 + .../no-changes-include-unchanged-plan.json | 39 + .../schema_registry/no-changes-output.txt | 3 + .../schema_registry/no-changes-plan.json | 5 + .../plans/schema_registry/no-changes.yaml | 24 + .../schema-registry-default-apply-output.txt | 29 + .../schema-registry-default-plan.json | 28 + .../schema-registry-default.yaml | 17 + .../schema-registry-mix-apply-output.txt | 42 + .../schema-registry-mix-plan.json | 39 + .../schema_registry/schema-registry-mix.yaml | 31 + .../schema-registry-new-avro-apply-output.txt | 16 + .../schema-registry-new-avro-plan.json | 17 + .../schema-registry-new-avro.yaml | 11 + .../schema-registry-new-json-apply-output.txt | 16 + .../schema-registry-new-json-plan.json | 17 + .../schema-registry-new-json.yaml | 11 + ...schema-registry-new-proto-apply-output.txt | 22 + .../schema-registry-new-proto-plan.json | 17 + .../schema-registry-new-proto.yaml | 11 + .../schemas/schema-registry-schema1.json | 8 + .../schemas/schema-registry-schema2.avsc | 11 + .../schemas/schema-registry-schema3.proto | 6 + ...eed-schema-modification-2-apply-output.txt | 23 + .../seed-schema-modification-2-plan.json | 28 + .../seed-schema-modification-2.yaml | 28 + ...eed-schema-modification-3-apply-output.txt | 17 + .../seed-schema-modification-3-plan.json | 22 + .../seed-schema-modification-3.yaml | 15 + ...eed-schema-modification-4-apply-output.txt | 20 + .../seed-schema-modification-4-plan.json | 23 + .../seed-schema-modification-4.yaml | 32 + .../seed-schema-modification-apply-output.txt | 50 + ...ma-modification-no-delete-apply-output.txt | 29 + ...ed-schema-modification-no-delete-plan.json | 17 + .../seed-schema-modification-no-delete.yaml | 15 + .../seed-schema-modification-plan.json | 43 + .../seed-schema-modification.yaml | 15 + .../resources/plans/seed-acl-exists-plan.json | 73 +- src/test/resources/plans/seed-acl-exists.yaml | 6 + .../seed-basic-include-unchanged-plan.json | 205 ++-- src/test/resources/plans/seed-basic-plan.json | 117 +-- src/test/resources/plans/seed-basic.yaml | 6 + .../plans/seed-blacklist-topics-plan.json | 69 +- .../plans/seed-blacklist-topics.yaml | 1 + .../plans/seed-topic-add-partitions-plan.json | 65 +- .../plans/seed-topic-add-partitions.yaml | 6 + .../plans/seed-topic-add-replicas-plan.json | 65 +- .../plans/seed-topic-add-replicas.json | 109 +-- .../plans/seed-topic-add-replicas.yaml | 6 + .../plans/seed-topic-modification-2-plan.json | 109 +-- .../plans/seed-topic-modification-2.yaml | 6 + .../plans/seed-topic-modification-3-plan.json | 109 +-- .../plans/seed-topic-modification-3.yaml | 6 + ...eed-topic-modification-no-delete-plan.json | 41 +- .../seed-topic-modification-no-delete.yaml | 6 + .../plans/seed-topic-modification-plan.json | 85 +- .../plans/seed-topic-modification.yaml | 6 + .../seed-topic-remove-replicas-plan.json | 1 + .../plans/seed-topic-remove-replicas.yaml | 6 + src/test/resources/plans/simple-plan.json | 25 +- .../resources/plans/simple-users-plan.json | 185 ++-- src/test/resources/plans/simple-users.yaml | 6 + src/test/resources/plans/simple.yaml | 6 + .../resources/plans/skip-acls-apply-plan.json | 161 +-- src/test/resources/plans/skip-acls-plan.json | 61 +- src/test/resources/plans/skip-acls.yaml | 6 + .../plans/topics-and-services-plan.json | 161 +-- .../resources/plans/topics-and-services.yaml | 6 + 149 files changed, 4482 insertions(+), 2676 deletions(-) create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasDefaults.java create mode 100644 src/main/java/com/devshawn/kafka/gitops/enums/SchemaCompatibility.java create mode 100644 src/test/resources/plans/schema_registry/add-with-reference.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-both-file-and-schema-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-both-file-and-schema.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-compatibility-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-compatibility.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-missing-compatibility-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-missing-compatibility.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-missing-file-and-schema-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-missing-file-and-schema.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-missing-type-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-missing-type.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-compatibility-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-compatibility.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-not-compatible.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-type-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-type.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-reference-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-reference.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-type-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-type.yaml create mode 100644 src/test/resources/plans/schema_registry/invalid-unrecognized-property-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-unrecognized-property.yaml create mode 100644 src/test/resources/plans/schema_registry/no-changes-include-unchanged-plan.json create mode 100644 src/test/resources/plans/schema_registry/no-changes-output.txt create mode 100644 src/test/resources/plans/schema_registry/no-changes-plan.json create mode 100644 src/test/resources/plans/schema_registry/no-changes.yaml create mode 100644 src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/schema-registry-default-plan.json create mode 100644 src/test/resources/plans/schema_registry/schema-registry-default.yaml create mode 100644 src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/schema-registry-mix-plan.json create mode 100644 src/test/resources/plans/schema_registry/schema-registry-mix.yaml create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-avro-plan.json create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-avro.yaml create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-json-plan.json create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-json.yaml create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-proto-plan.json create mode 100644 src/test/resources/plans/schema_registry/schema-registry-new-proto.yaml create mode 100644 src/test/resources/plans/schema_registry/schemas/schema-registry-schema1.json create mode 100644 src/test/resources/plans/schema_registry/schemas/schema-registry-schema2.avsc create mode 100644 src/test/resources/plans/schema_registry/schemas/schema-registry-schema3.proto create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-2-plan.json create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-3-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-3-plan.json create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-3.yaml create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-4-plan.json create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-plan.json create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-no-delete.yaml create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-plan.json create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification.yaml diff --git a/.gitignore b/.gitignore index 1cbbcbf7..5820d6be 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ state.yaml plan.json test.py /generated/ +/.apt_generated/ +/.apt_generated_tests/ diff --git a/Dockerfile b/Dockerfile index cedf3942..c8f7f8c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,11 @@ -FROM openjdk:8-jre-slim +FROM gradle:5.6.4-jdk8 AS build +COPY --chown=gradle:gradle . /home/gradle/src +WORKDIR /home/gradle/src +RUN gradle clean build buildRelease -x test +FROM openjdk:8-jre-slim RUN apt-get update && apt-get --yes upgrade && \ apt-get install -y python3 python3-pip curl && \ rm -rf /var/lib/apt/lists/* -COPY ./build/output/kafka-gitops /usr/local/bin/kafka-gitops \ No newline at end of file +COPY --from=build /home/gradle/src/build/output/kafka-gitops /usr/local/bin/kafka-gitops diff --git a/build.gradle b/build.gradle index 23d95f37..f3d9d024 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,12 @@ dependencies { compile 'info.picocli:picocli:4.1.4' implementation ('io.confluent:kafka-schema-registry-client:6.1.1') +<<<<<<< HEAD implementation('com.flipkart.zjsonpatch:zjsonpatch:0.4.11') +======= + implementation ('io.confluent:kafka-json-schema-provider:6.1.1') + implementation ('io.confluent:kafka-protobuf-serializer:6.1.1') +>>>>>>> a208542 (Tests) compile 'org.slf4j:slf4j-api:1.7.30' compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2d5ae4f2..094ba014 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -92,3 +92,24 @@ services: depends_on: - zoo1 + schema-registry: + image: confluentinc/cp-schema-registry:6.1.1 + hostname: schema-registry + ports: + - "8082:8082" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka1:19092,kafka2:19092,kafka3:19092" + SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT + SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN + SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8082" + SCHEMA_REGISTRY_GROUP_ID: "schema-registry-test" + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/registry_jaas.conf" + SCHEMA_REGISTRY_OPTS: "-Djava.security.auth.login.config=/etc/kafka/registry_jaas.conf" + volumes: + - ./config/registry_jaas.conf:/etc/kafka/registry_jaas.conf + depends_on: + - kafka1 + - kafka2 + - kafka3 + diff --git a/src/main/java/com/devshawn/kafka/gitops/StateManager.java b/src/main/java/com/devshawn/kafka/gitops/StateManager.java index cf747385..c03c6afb 100644 --- a/src/main/java/com/devshawn/kafka/gitops/StateManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/StateManager.java @@ -1,9 +1,16 @@ package com.devshawn.kafka.gitops; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.LoggerFactory; import com.devshawn.kafka.gitops.config.KafkaGitopsConfigLoader; import com.devshawn.kafka.gitops.config.ManagerConfig; +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; import com.devshawn.kafka.gitops.domain.confluent.ServiceAccount; import com.devshawn.kafka.gitops.domain.options.GetAclOptions; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; @@ -11,8 +18,10 @@ import com.devshawn.kafka.gitops.domain.state.CustomAclDetails; import com.devshawn.kafka.gitops.domain.state.DesiredState; import com.devshawn.kafka.gitops.domain.state.DesiredStateFile; +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; import com.devshawn.kafka.gitops.domain.state.TopicDetails; import com.devshawn.kafka.gitops.domain.state.service.KafkaStreamsService; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import com.devshawn.kafka.gitops.exception.ConfluentCloudException; import com.devshawn.kafka.gitops.exception.InvalidAclDefinitionException; import com.devshawn.kafka.gitops.exception.MissingConfigurationException; @@ -24,30 +33,26 @@ import com.devshawn.kafka.gitops.service.KafkaService; import com.devshawn.kafka.gitops.service.ParserService; import com.devshawn.kafka.gitops.service.RoleService; +import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.LogUtil; import com.devshawn.kafka.gitops.util.StateUtil; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; public class StateManager { - private static org.slf4j.Logger log = LoggerFactory.getLogger(StateManager.class); - private final ManagerConfig managerConfig; private final ObjectMapper objectMapper; private final ParserService parserService; private final KafkaService kafkaService; + private final SchemaRegistryService schemaRegistryService; private final RoleService roleService; private final ConfluentCloudService confluentCloudService; @@ -163,7 +168,7 @@ private DesiredState getDesiredState() { } private void generateTopicsState(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { - Optional defaultReplication = StateUtil.fetchReplication(desiredStateFile); + Optional defaultReplication = StateUtil.fetchDefaultTopicsReplication(desiredStateFile); if (defaultReplication.isPresent()) { desiredStateFile.getTopics().forEach((name, details) -> { Integer replication = details.getReplication().isPresent() ? details.getReplication().get() : defaultReplication.get(); @@ -175,7 +180,15 @@ private void generateTopicsState(DesiredState.Builder desiredState, DesiredState } private void generateSchemasState(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { - desiredState.putAllSchemas(desiredStateFile.getSchemas()); + Optional defaultSchemaCompatibility = StateUtil.fetchDefaultSchemasCompatibility(desiredStateFile); + if (defaultSchemaCompatibility.isPresent()) { + desiredStateFile.getSchemas().forEach((s, details) -> { + SchemaCompatibility compatibility = details.getCompatibility().isPresent() ? details.getCompatibility().get() : defaultSchemaCompatibility.get(); + desiredState.putSchemas(s, new SchemaDetails.Builder().mergeFrom(details).setCompatibility(compatibility).build()); + }); + } else { + desiredState.putAllSchemas(desiredStateFile.getSchemas()); + } } private void generateConfluentCloudServiceAcls(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) { @@ -316,7 +329,7 @@ private void validateCustomAcls(DesiredStateFile desiredStateFile) { } private void validateTopics(DesiredStateFile desiredStateFile) { - Optional defaultReplication = StateUtil.fetchReplication(desiredStateFile); + Optional defaultReplication = StateUtil.fetchDefaultTopicsReplication(desiredStateFile); if (!defaultReplication.isPresent()) { desiredStateFile.getTopics().forEach((name, details) -> { if (!details.getReplication().isPresent()) { @@ -331,42 +344,13 @@ private void validateTopics(DesiredStateFile desiredStateFile) { } private void validateSchemas(DesiredStateFile desiredStateFile) { - if (!desiredStateFile.getSchemas().isEmpty()) { - SchemaRegistryConfig schemaRegistryConfig = SchemaRegistryConfigLoader.load(); - desiredStateFile.getSchemas().forEach((s, schemaDetails) -> { - if (!schemaDetails.getType().equalsIgnoreCase("Avro")) { - throw new ValidationException(String.format("Schema type %s is currently not supported.", schemaDetails.getType())); - } - if (!Files.exists(Paths.get(schemaRegistryConfig.getConfig().get("SCHEMA_DIRECTORY") + "/" + schemaDetails.getFile()))) { - throw new ValidationException(String.format("Schema file %s not found in schema directory at %s", schemaDetails.getFile(), schemaRegistryConfig.getConfig().get("SCHEMA_DIRECTORY"))); - } - if (schemaDetails.getType().equalsIgnoreCase("Avro")) { - AvroSchemaProvider avroSchemaProvider = new AvroSchemaProvider(); - if (schemaDetails.getReferences().isEmpty() && schemaDetails.getType().equalsIgnoreCase("Avro")) { - Optional parsedSchema = avroSchemaProvider.parseSchema(schemaRegistryService.loadSchemaFromDisk(schemaDetails.getFile()), Collections.emptyList()); - if (!parsedSchema.isPresent()) { - throw new ValidationException(String.format("Avro schema %s could not be parsed.", schemaDetails.getFile())); - } - } else { - List schemaReferences = new ArrayList<>(); - schemaDetails.getReferences().forEach(referenceDetails -> { - SchemaReference schemaReference = new SchemaReference(referenceDetails.getName(), referenceDetails.getSubject(), referenceDetails.getVersion()); - schemaReferences.add(schemaReference); - }); - // we need to pass a schema registry client as a config because the underlying code validates against the current state - avroSchemaProvider.configure(Collections.singletonMap(SchemaProvider.SCHEMA_VERSION_FETCHER_CONFIG, schemaRegistryService.createSchemaRegistryClient())); - try { - Optional parsedSchema = avroSchemaProvider.parseSchema(schemaRegistryService.loadSchemaFromDisk(schemaDetails.getFile()), schemaReferences); - if (!parsedSchema.isPresent()) { - throw new ValidationException(String.format("Avro schema %s could not be parsed.", schemaDetails.getFile())); - } - } catch (IllegalStateException ex) { - throw new ValidationException(String.format("Reference validation error: %s", ex.getMessage())); - } catch (RuntimeException ex) { - throw new ValidationException(String.format("Error thrown when attempting to validate schema with reference", ex.getMessage())); - } - } + Optional defaultSchemaCompatibility = StateUtil.fetchDefaultSchemasCompatibility(desiredStateFile); + if (!defaultSchemaCompatibility.isPresent()) { + desiredStateFile.getSchemas().forEach((subject, details) -> { + if (!details.getCompatibility().isPresent()) { + throw new ValidationException(String.format("Not set: [compatibility] in state file definition: schema -> %s", subject)); } + schemaRegistryService.validateSchema(subject, details); }); } } @@ -379,11 +363,17 @@ private boolean isConfluentCloudEnabled(DesiredStateFile desiredStateFile) { } private ObjectMapper initializeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - objectMapper.registerModule(new Jdk8Module()); - return objectMapper; + ObjectMapper gitopsObjectMapper = new ObjectMapper(); + gitopsObjectMapper.enable(SerializationFeature.INDENT_OUTPUT); + gitopsObjectMapper.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES); + gitopsObjectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + gitopsObjectMapper.registerModule(new Jdk8Module()); + DefaultIndenter defaultIndenter = new DefaultIndenter(" ", DefaultIndenter.SYS_LF); + DefaultPrettyPrinter printer = new DefaultPrettyPrinter() + .withObjectIndenter(defaultIndenter) + .withArrayIndenter(defaultIndenter); + gitopsObjectMapper.setDefaultPrettyPrinter(printer); + return gitopsObjectMapper; } private void initializeLogger(boolean verbose) { diff --git a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java index e2497dc7..d652b982 100644 --- a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java +++ b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java @@ -12,6 +12,14 @@ public class SchemaRegistryConfigLoader { private static org.slf4j.Logger log = LoggerFactory.getLogger(SchemaRegistryConfigLoader.class); + public static final String SCHEMA_REGISTRY_URL_KEY = "SCHEMA_REGISTRY_URL"; + public static final String SCHEMA_DIRECTORY_KEY = "SCHEMA_DIRECTORY"; + public static final String SCHEMA_REGISTRY_SASL_JAAS_USERNAME_KEY = "SCHEMA_REGISTRY_SASL_JAAS_USERNAME"; + public static final String SCHEMA_REGISTRY_SASL_JAAS_PASSWORD_KEY = "SCHEMA_REGISTRY_SASL_JAAS_PASSWORD"; + public static final String SCHEMA_REGISTRY_SASL_CONFIG_KEY = "SCHEMA_REGISTRY_SASL_CONFIG"; + + private SchemaRegistryConfigLoader() {} + public static SchemaRegistryConfig load() { SchemaRegistryConfig.Builder builder = new SchemaRegistryConfig.Builder(); setConfig(builder); @@ -26,14 +34,14 @@ private static void setConfig(SchemaRegistryConfig.Builder builder) { Map environment = System.getenv(); environment.forEach((key, value) -> { - if (key.equals("SCHEMA_REGISTRY_SASL_JAAS_USERNAME")) { + if (key.equals(SCHEMA_REGISTRY_SASL_JAAS_USERNAME_KEY)) { username.set(value); - } else if (key.equals("SCHEMA_REGISTRY_SASL_JAAS_PASSWORD")) { + } else if (key.equals(SCHEMA_REGISTRY_SASL_JAAS_PASSWORD_KEY)) { password.set(value); - } else if (key.equals("SCHEMA_REGISTRY_URL")) { - config.put("SCHEMA_REGISTRY_URL", value); - } else if (key.equals("SCHEMA_DIRECTORY")) { - config.put("SCHEMA_DIRECTORY", value); + } else if (key.equals(SCHEMA_REGISTRY_URL_KEY)) { + config.put(SCHEMA_REGISTRY_URL_KEY, value); + } else if (key.equals(SCHEMA_DIRECTORY_KEY)) { + config.put(SCHEMA_DIRECTORY_KEY, value); } }); @@ -48,13 +56,13 @@ private static void setConfig(SchemaRegistryConfig.Builder builder) { private static void handleDefaultConfig(Map config) { final String DEFAULT_URL = "http://localhost:8081"; final String CURRENT_WORKING_DIR = System.getProperty("user.dir"); - if (!config.containsKey("SCHEMA_REGISTRY_URL")) { - log.info("SCHEMA_REGISTRY_URL not set. Using default value of {}", DEFAULT_URL); - config.put("SCHEMA_REGISTRY_URL", DEFAULT_URL); + if (!config.containsKey(SCHEMA_REGISTRY_URL_KEY)) { + log.info("{} not set. Using default value of {}", SCHEMA_REGISTRY_URL_KEY, DEFAULT_URL); + config.put(SCHEMA_REGISTRY_URL_KEY, DEFAULT_URL); } - if (!config.containsKey("SCHEMA_DIRECTORY")) { - log.info("SCHEMA_DIRECTORY not set. Defaulting to current working directory: {}", CURRENT_WORKING_DIR); - config.put("SCHEMA_DIRECTORY", CURRENT_WORKING_DIR); + if (!config.containsKey(SCHEMA_DIRECTORY_KEY)) { + log.info("{} not set. Defaulting to current working directory: {}", SCHEMA_DIRECTORY_KEY, CURRENT_WORKING_DIR); + config.put(SCHEMA_DIRECTORY_KEY, CURRENT_WORKING_DIR); } } @@ -63,13 +71,12 @@ private static void handleAuthentication(AtomicReference username, Atomi String loginModule = "org.apache.kafka.common.security.plain.PlainLoginModule"; String value = String.format("%s required username=\"%s\" password=\"%s\";", loginModule, escape(username.get()), escape(password.get())); - config.put("SCHEMA_REGISTRY_SASL_CONFIG", value); - } else if (username.get() != null) { - throw new MissingConfigurationException("SCHEMA_REGISTRY_SASL_JAAS_USERNAME"); - } else if (password.get() != null) { - throw new MissingConfigurationException("SCHEMA_REGISTRY_SASL_JAAS_PASSWORD"); - } else if (username.get() == null & password.get() == null) { - throw new MissingMultipleConfigurationException("SCHEMA_REGISTRY_SASL_JAAS_PASSWORD", "SCHEMA_REGISTRY_SASL_JAAS_USERNAME"); + config.put(SCHEMA_REGISTRY_SASL_CONFIG_KEY, value); + } else { + if(config.get(SCHEMA_REGISTRY_SASL_CONFIG_KEY) == null) { + log.info("{} or {} not set. No authentication configured for the Schema Registry", + SCHEMA_REGISTRY_SASL_JAAS_USERNAME_KEY, SCHEMA_REGISTRY_SASL_JAAS_PASSWORD_KEY); + } } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java index 47f5a9d7..210d4fa1 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java @@ -1,22 +1,79 @@ package com.devshawn.kafka.gitops.domain.state; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import org.inferred.freebuilder.FreeBuilder; - +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.inferred.freebuilder.FreeBuilder; +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; +import com.devshawn.kafka.gitops.enums.SchemaType; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; +import com.devshawn.kafka.gitops.exception.ValidationException; +import com.devshawn.kafka.gitops.service.SchemaRegistryService; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.confluent.kafka.schemaregistry.AbstractSchemaProvider; +import io.confluent.kafka.schemaregistry.ParsedSchema; @FreeBuilder -@JsonDeserialize(builder = SchemaDetails.Builder.class) +@JsonDeserialize(builder = SchemaDetails.Builder.class) public interface SchemaDetails { - String getType(); + SchemaType getType(); + + String getSchema(); - String getFile(); + Optional getFile(); - List getSubjects(); + Optional getCompatibility(); List getReferences(); class Builder extends SchemaDetails_Builder { + @Override + public SchemaDetails build() { + AbstractSchemaProvider schemaProvider = SchemaRegistryService.schemaProviderFromType(super.getType()); + ParsedSchema parsedSchema; + if(super.getFile().isPresent()) { + boolean schema = true; + try { + super.getSchema(); + }catch (IllegalStateException e) { + schema = false; + } + if ( schema ) { + throw new IllegalStateException("schema and file fields cannot be both set at the same time"); + } + parsedSchema = schemaProvider.parseSchema(loadSchemaFromDisk(super.getFile().get()), Collections.emptyList()).get(); + super.setFile(Optional.empty()); + } else { + String schema; + try { + schema = super.getSchema(); + }catch (IllegalStateException e) { + throw new IllegalStateException("schema or file field must be provided"); + } + parsedSchema = schemaProvider.parseSchema(schema, Collections.emptyList()).get(); + } + super.setSchema(parsedSchema.canonicalString()); + return super.build(); + } + + private String loadSchemaFromDisk(String fileName) { + Map config = SchemaRegistryConfigLoader.load().getConfig(); + final String SCHEMA_DIRECTORY = config.get(SchemaRegistryConfigLoader.SCHEMA_DIRECTORY_KEY).toString(); + if (!Files.exists(Paths.get(SCHEMA_DIRECTORY + "/" + fileName))) { + throw new ValidationException(String.format("Schema file %s not found in schema directory at %s", getFile(), config.get("SCHEMA_DIRECTORY"))); + } + try { + return new String(Files.readAllBytes(Paths.get(SCHEMA_DIRECTORY + "/" + fileName)), StandardCharsets.UTF_8); + } catch (IOException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to load a schema from schema directory", ex.getMessage()); + } + } } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java index 49494924..ee314fb1 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java @@ -14,6 +14,8 @@ public interface SettingsFiles { Optional getTopics(); Optional getUsers(); + + Optional getSchemas(); class Builder extends SettingsFiles_Builder { } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java index 7e0eab3e..0958915e 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java @@ -1,5 +1,6 @@ package com.devshawn.kafka.gitops.domain.state.settings; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.inferred.freebuilder.FreeBuilder; @@ -13,6 +14,8 @@ public interface SettingsSchema { Optional getDirectory(); + Optional getDefaults(); + class Builder extends SettingsSchema_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasDefaults.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasDefaults.java new file mode 100644 index 00000000..db60b91c --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasDefaults.java @@ -0,0 +1,16 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import java.util.Optional; +import org.inferred.freebuilder.FreeBuilder; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@FreeBuilder +@JsonDeserialize(builder = SettingsSchemasDefaults.Builder.class) +public interface SettingsSchemasDefaults { + + Optional getCompatibility(); + + class Builder extends SettingsSchemasDefaults_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/enums/SchemaCompatibility.java b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaCompatibility.java new file mode 100644 index 00000000..34fae822 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaCompatibility.java @@ -0,0 +1,11 @@ +package com.devshawn.kafka.gitops.enums; + +public enum SchemaCompatibility { + NONE, + BACKWARD, + FORWARD, + FULL, + BACKWARD_TRANSITIVE, + FORWARD_TRANSITIVE, + FULL_TRANSITIVE; +} diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java index 12a28665..7568e8cf 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java @@ -1,5 +1,14 @@ package com.devshawn.kafka.gitops.manager; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.kafka.clients.admin.AlterConfigOp; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.config.ConfigResource; import com.devshawn.kafka.gitops.config.ManagerConfig; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; import com.devshawn.kafka.gitops.domain.plan.TopicConfigPlan; @@ -9,12 +18,6 @@ import com.devshawn.kafka.gitops.service.KafkaService; import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.LogUtil; -import org.apache.kafka.clients.admin.AlterConfigOp; -import org.apache.kafka.clients.admin.ConfigEntry; -import org.apache.kafka.common.Node; -import org.apache.kafka.common.config.ConfigResource; - -import java.util.*; public class ApplyManager { diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java index 0dbf1543..c04df2b4 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java @@ -1,5 +1,19 @@ package com.devshawn.kafka.gitops.manager; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.config.ConfigResource; +import org.slf4j.LoggerFactory; import com.devshawn.kafka.gitops.config.ManagerConfig; import com.devshawn.kafka.gitops.domain.plan.AclPlan; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; @@ -12,6 +26,7 @@ import com.devshawn.kafka.gitops.domain.state.DesiredState; import com.devshawn.kafka.gitops.domain.state.TopicDetails; import com.devshawn.kafka.gitops.enums.PlanAction; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; import com.devshawn.kafka.gitops.exception.ReadPlanInputException; import com.devshawn.kafka.gitops.exception.ValidationException; @@ -21,21 +36,6 @@ import com.devshawn.kafka.gitops.util.PlanUtil; import com.fasterxml.jackson.databind.ObjectMapper; import io.confluent.kafka.schemaregistry.client.SchemaMetadata; -import org.apache.kafka.clients.admin.Config; -import org.apache.kafka.clients.admin.ConfigEntry; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.acl.AclBinding; -import org.apache.kafka.common.config.ConfigResource; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; public class PlanManager { @@ -228,10 +228,17 @@ public void planAcls(DesiredState desiredState, DesiredPlan.Builder desiredPlan) public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPlan) { // TODO: Parallelize getting schema metadata? Map currentSubjectSchemasMap = new HashMap<>(); - schemaRegistryService.getAllSubjects().forEach(subject -> { + Map currentSubjectCompatibilityMap = new HashMap<>(); + List subjects = schemaRegistryService.getAllSubjects(); + subjects.forEach(subject -> { SchemaMetadata schemaMetadata = schemaRegistryService.getLatestSchemaMetadata(subject); currentSubjectSchemasMap.put(subject, schemaMetadata); }); + SchemaCompatibility globalCompatibility = schemaRegistryService.getGlobalSchemaCompatibility(); + subjects.forEach(subject -> { + SchemaCompatibility compatibility = schemaRegistryService.getSchemaCompatibility(subject, globalCompatibility); + currentSubjectCompatibilityMap.put(subject, compatibility); + }); desiredState.getSchemas().forEach((subject, schemaDetails) -> { SchemaPlan.Builder schemaPlan = new SchemaPlan.Builder() @@ -239,15 +246,30 @@ public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPl .setSchemaDetails(schemaDetails); if (!currentSubjectSchemasMap.containsKey(subject)) { - log.info("[PLAN] Schema Subject {} does not exist; it will be created.", subject); + log.info("[PLAN] Schema Subject '{}' does not exist; it will be created.", subject); schemaPlan.setAction(PlanAction.ADD); } else { - String diff = schemaRegistryService.compareSchemasAndReturnDiff(schemaRegistryService.loadSchemaFromDisk(schemaDetails.getFile()), currentSubjectSchemasMap.get(subject).getSchema()); - if (diff == null) { - log.info("[PLAN] Schema Subject {} exists and has not changed; it will not be created.", subject); + SchemaMetadata currentSchema = currentSubjectSchemasMap.get(subject); + if(! schemaDetails.getType().toString().equals(currentSchema.getSchemaType())) { + throw new ValidationException("Changing the schema type is not allowed " + + "(subject: " + subject + + ", current type: " + currentSchema.getSchemaType() + + ", new type:"+schemaDetails.getType()+")"); + } + SchemaCompatibility currentCompatibility = currentSubjectCompatibilityMap.get(subject); + if( schemaDetails.getCompatibility().get() != currentCompatibility ) { + // TODO: we should be able to do this + throw new ValidationException("Changing the subject compatibility is not allowed with kafka-gitops" + + "(subject: " + subject + + ", current compatibilty: " + currentCompatibility + + ", new compatibilty:" + schemaDetails.getCompatibility().get() + ")"); + } + boolean diff = schemaRegistryService.deepEquals(schemaDetails, currentSubjectSchemasMap.get(subject)); + if (diff) { + log.info("[PLAN] Schema Subject '{}' exists and has not changed; it will not be created.", subject); schemaPlan.setAction(PlanAction.NO_CHANGE); } else { - log.info("[PLAN] Schema Subject {} exists and has changed; it will be updated.", subject); + log.info("[PLAN] Schema Subject '{}' exists and has changed; it will be updated.", subject); schemaPlan.setAction(PlanAction.UPDATE); // TODO: Set diff string for logging? } @@ -258,6 +280,7 @@ public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPl currentSubjectSchemasMap.forEach((subject, schemaMetadata) -> { if (!managerConfig.isDeleteDisabled() && desiredState.getSchemas().getOrDefault(subject, null) == null) { + log.info("[PLAN] Schema Subject '{}' exists and will be remove.", subject); SchemaPlan schemaPlan = new SchemaPlan.Builder() .setName(subject) .setAction(PlanAction.REMOVE) diff --git a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java index 80b9c682..9fe9a710 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java @@ -55,6 +55,10 @@ public DesiredStateFile parseStateFile() { DesiredStateFile usersFile = loadExternalFile(settingsFiles.getUsers().get(), "Users"); builder.putAllUsers(usersFile.getUsers()); } + if (settingsFiles.getSchemas().isPresent()) { + DesiredStateFile schemasFile = loadExternalFile(settingsFiles.getSchemas().get(), "Schemas"); + builder.putAllSchemas(schemasFile.getSchemas()); + } return builder.build(); } return desiredStateFile; diff --git a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java index 1793d5d1..4f78c056 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java @@ -1,27 +1,33 @@ package com.devshawn.kafka.gitops.service; - +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.kafka.common.config.SaslConfigs; import com.devshawn.kafka.gitops.config.SchemaRegistryConfig; +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; +import com.devshawn.kafka.gitops.enums.SchemaType; import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.zjsonpatch.JsonDiff; +import com.devshawn.kafka.gitops.exception.ValidationException; +import io.confluent.kafka.schemaregistry.AbstractSchemaProvider; import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.kafka.schemaregistry.SchemaProvider; import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider; import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.SchemaMetadata; import io.confluent.kafka.schemaregistry.client.rest.RestService; +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import io.confluent.kafka.schemaregistry.client.security.basicauth.SaslBasicAuthCredentialProvider; -import org.apache.kafka.common.config.SaslConfigs; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; +import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider; public class SchemaRegistryService { @@ -31,6 +37,10 @@ public SchemaRegistryService(SchemaRegistryConfig config) { this.config = config; } + public Map getConfig() { + return Collections.unmodifiableMap(config.getConfig()); + } + public List getAllSubjects() { final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); try { @@ -55,13 +65,81 @@ public void deleteSubject(String subject, boolean isPermanent) { public int register(SchemaPlan schemaPlan) { final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); - AvroSchemaProvider avroSchemaProvider = new AvroSchemaProvider(); - Optional parsedSchema = avroSchemaProvider.parseSchema(loadSchemaFromDisk(schemaPlan.getSchemaDetails().get().getFile()), Collections.emptyList()); + AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaPlan.getSchemaDetails().get().getType()); + ParsedSchema parsedSchema = parseSchema(schemaPlan.getName(), schemaPlan.getSchemaDetails().get(), schemaProvider, cachedSchemaRegistryClient); + int id; + try { + id = cachedSchemaRegistryClient.register(schemaPlan.getName(), parsedSchema); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to register subject '" + schemaPlan.getName() + "' in schema registry", ex.getMessage()); + } + try { + cachedSchemaRegistryClient.updateCompatibility(schemaPlan.getName(), schemaPlan.getSchemaDetails().get().getCompatibility().get().toString()); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to update compatibility of the newly registered subject '" + schemaPlan.getName() + "' in schema registry", ex.getMessage()); + } + return id; + } + + public static AbstractSchemaProvider schemaProviderFromType(SchemaType schemaType) { + AbstractSchemaProvider schemaProvider; + if (schemaType == SchemaType.AVRO) { + schemaProvider = new AvroSchemaProvider(); + } else if (schemaType == SchemaType.JSON) { + schemaProvider = new JsonSchemaProvider(); + } else if (schemaType == SchemaType.PROTOBUF) { + schemaProvider = new ProtobufSchemaProvider(); + } else { + throw new ValidationException("Unknown schema type: " + schemaType); + } + return schemaProvider; + } + + public void validateSchema(String subject, SchemaDetails schemaDetails) { + AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaDetails.getType()); + validateSchema(subject, schemaDetails, schemaProvider); + } + + public void validateSchema(String subject, SchemaDetails schemaDetails, AbstractSchemaProvider schemaProvider) { + CachedSchemaRegistryClient schemaRegistryClient = createSchemaRegistryClient(); + ParsedSchema parsedSchema = parseSchema(subject, schemaDetails, schemaProvider, schemaRegistryClient); try { - return cachedSchemaRegistryClient.register(schemaPlan.getName(), parsedSchema.get()); + List differences = schemaRegistryClient.testCompatibilityVerbose(subject, parsedSchema); + if(! differences.isEmpty()) { + throw new ValidationException(String.format("%s schema '%s' is not compatible with the latest one: %s", schemaProvider.schemaType(), subject, differences)); + } } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to register subject with schema registry", ex.getMessage()); + throw new ValidationException(String.format("Error thrown when attempting to check the compatibility of the new schema for '%s': %s", subject, ex.getMessage())); + } + } + + private ParsedSchema parseSchema(String subject, SchemaDetails schemaDetails, AbstractSchemaProvider schemaProvider, CachedSchemaRegistryClient schemaRegistryClient) { + Optional parsedSchema; + if (schemaDetails.getReferences().isEmpty()) { + parsedSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), Collections.emptyList()); + if (!parsedSchema.isPresent()) { + throw new ValidationException(String.format("%s schema for subject '%s' could not be parsed.", schemaProvider.schemaType(), subject)); + } + } else { + List schemaReferences = new ArrayList<>(); + schemaDetails.getReferences().forEach(referenceDetails -> { + SchemaReference schemaReference = new SchemaReference(referenceDetails.getName(), referenceDetails.getSubject(), referenceDetails.getVersion()); + schemaReferences.add(schemaReference); + }); + // we need to pass a schema registry client as a config because the underlying code validates against the current state + schemaProvider.configure(Collections.singletonMap(SchemaProvider.SCHEMA_VERSION_FETCHER_CONFIG, schemaRegistryClient)); + try { + parsedSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), schemaReferences); + } catch (IllegalStateException ex) { + throw new ValidationException(String.format("Reference validation error: %s", ex.getMessage())); + } catch (RuntimeException ex) { + throw new ValidationException(String.format("Error thrown when attempting to validate %s schema with reference: %s", subject, ex.getMessage())); + } + if (!parsedSchema.isPresent()) { + throw new ValidationException(String.format("%s referenced schema could not be parsed for subject %s", schemaProvider.schemaType(), subject)); + } } + return parsedSchema.get(); } public SchemaMetadata getLatestSchemaMetadata(String subject) { @@ -69,41 +147,49 @@ public SchemaMetadata getLatestSchemaMetadata(String subject) { try { return cachedSchemaRegistryClient.getLatestSchemaMetadata(subject); } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); + throw new SchemaRegistryExecutionException("Error thrown when attempting to get schema metadata for subject '" + subject + "'", ex.getMessage()); } } - public String compareSchemasAndReturnDiff(String schemaStringOne, String schemaStringTwo) { - ObjectMapper objectMapper = new ObjectMapper(); + public SchemaCompatibility getGlobalSchemaCompatibility() { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); try { - JsonNode schemaOne = objectMapper.readTree(schemaStringOne); - JsonNode schemaTwo = objectMapper.readTree(schemaStringTwo); - JsonNode diff = JsonDiff.asJson(schemaOne, schemaTwo); - if (diff.isEmpty()) { - return null; - } - return diff.toString(); - } catch (JsonProcessingException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to compare the schemas", ex.getMessage()); + return SchemaCompatibility.valueOf(cachedSchemaRegistryClient.getCompatibility(null)); + } catch (IOException | RestClientException ex) { + throw new SchemaRegistryExecutionException("Error thrown when attempting to get global schema compatibility", ex.getMessage()); } } - - public String loadSchemaFromDisk(String fileName) { - final String SCHEMA_DIRECTORY = config.getConfig().get("SCHEMA_DIRECTORY").toString(); + public SchemaCompatibility getSchemaCompatibility(String subject, SchemaCompatibility globalCompatibility) { + final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); try { - return new String(Files.readAllBytes(Paths.get(SCHEMA_DIRECTORY + "/" + fileName)), StandardCharsets.UTF_8); + return SchemaCompatibility.valueOf(cachedSchemaRegistryClient.getCompatibility(subject)); } catch (IOException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to load a schema from schema directory", ex.getMessage()); + throw new SchemaRegistryExecutionException("Error thrown when attempting to get schema compatibility for subject '" + subject + "'", ex.getMessage()); + } catch (RestClientException ex) { + if(ex.getErrorCode() == 40401) { + return globalCompatibility; + } + throw new SchemaRegistryExecutionException("Error thrown when attempting to get schema compatibility for subject '" + subject + "'", ex.getMessage()); } } + public boolean deepEquals(SchemaDetails schemaDetails, SchemaMetadata schemaMetadata) { + AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaDetails.getType()); + ParsedSchema newSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), Collections.emptyList()).get(); + ParsedSchema previousSchema = schemaProvider.parseSchema(schemaMetadata.getSchema(), Collections.emptyList()).get(); + return (newSchema.deepEquals(previousSchema)); + } + public CachedSchemaRegistryClient createSchemaRegistryClient() { - RestService restService = new RestService(config.getConfig().get("SCHEMA_REGISTRY_URL").toString()); - SaslBasicAuthCredentialProvider saslBasicAuthCredentialProvider = new SaslBasicAuthCredentialProvider(); - Map clientConfig = new HashMap<>(); - clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, config.getConfig().get("SCHEMA_REGISTRY_SASL_CONFIG").toString()); - saslBasicAuthCredentialProvider.configure(clientConfig); - restService.setBasicAuthCredentialProvider(saslBasicAuthCredentialProvider); + RestService restService = new RestService(config.getConfig().get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()); + if(config.getConfig().get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY) != null) { + SaslBasicAuthCredentialProvider saslBasicAuthCredentialProvider = new SaslBasicAuthCredentialProvider(); + Map clientConfig = new HashMap<>(); + clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, config.getConfig() + .get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY).toString()); + saslBasicAuthCredentialProvider.configure(clientConfig); + restService.setBasicAuthCredentialProvider(saslBasicAuthCredentialProvider); + } return new CachedSchemaRegistryClient(restService, 10); } diff --git a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java index 352abfc8..2fb78754 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java @@ -155,10 +155,14 @@ private static void printSchemaPlan(SchemaPlan schemaPlan) { case ADD: System.out.println(green(String.format("+ [SCHEMA] %s", schemaPlan.getName()))); System.out.println(green(String.format("\t + type: %s", schemaPlan.getSchemaDetails().get().getType()))); - System.out.println(green(String.format("\t + file: %s", schemaPlan.getSchemaDetails().get().getFile()))); + if(schemaPlan.getSchemaDetails().get().getCompatibility().isPresent()) { + System.out.println(green(String.format("\t + compatibility: %s", schemaPlan.getSchemaDetails().get().getCompatibility().get()))); + } + System.out.println(green(String.format("\t + schema: \n----------------------\n%s\n----------------------", + schemaPlan.getSchemaDetails().get().getSchema()))); if (!schemaPlan.getSchemaDetails().get().getReferences().isEmpty()) { schemaPlan.getSchemaDetails().get().getReferences().forEach(referenceDetail -> { - System.out.println(green(String.format("\t + reference:"))); + System.out.println(green(String.format("\t + references:"))); System.out.println(green(String.format("\t\t + name: %s", referenceDetail.getName()))); System.out.println(green(String.format("\t\t + subject: %s", referenceDetail.getSubject()))); System.out.println(green(String.format("\t\t + version: %s", referenceDetail.getVersion()))); diff --git a/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java index 34879b9e..ffc72869 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java @@ -1,12 +1,12 @@ package com.devshawn.kafka.gitops.util; import com.devshawn.kafka.gitops.domain.state.DesiredStateFile; - +import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import java.util.Optional; public class StateUtil { - public static Optional fetchReplication(DesiredStateFile desiredStateFile) { + public static Optional fetchDefaultTopicsReplication(DesiredStateFile desiredStateFile) { if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getTopics().isPresent() && desiredStateFile.getSettings().get().getTopics().get().getDefaults().isPresent()) { return desiredStateFile.getSettings().get().getTopics().get().getDefaults().get().getReplication(); @@ -20,4 +20,12 @@ public static boolean isDescribeTopicAclEnabled(DesiredStateFile desiredStateFil && desiredStateFile.getSettings().get().getServices().get().getAcls().get().getDescribeTopicEnabled().isPresent() && desiredStateFile.getSettings().get().getServices().get().getAcls().get().getDescribeTopicEnabled().get(); } + + public static Optional fetchDefaultSchemasCompatibility(DesiredStateFile desiredStateFile) { + if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getSchema().isPresent() + && desiredStateFile.getSettings().get().getSchema().get().getDefaults().isPresent()) { + return desiredStateFile.getSettings().get().getSchema().get().getDefaults().get().getCompatibility(); + } + return Optional.empty(); + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy index 130d4ecc..13fa1ce7 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy @@ -1,5 +1,7 @@ package com.devshawn.kafka.gitops +import java.nio.file.Path +import java.nio.file.Paths import org.junit.Rule import org.junit.contrib.java.lang.system.EnvironmentVariables import picocli.CommandLine @@ -13,16 +15,20 @@ class ApplyCommandIntegrationSpec extends Specification { EnvironmentVariables environmentVariables void setup() { - environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") + environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092,localhost:9093,localhost:9094") environmentVariables.set("KAFKA_SASL_JAAS_USERNAME", "test") environmentVariables.set("KAFKA_SASL_JAAS_PASSWORD", "test-secret") environmentVariables.set("KAFKA_SASL_MECHANISM", "PLAIN") environmentVariables.set("KAFKA_SECURITY_PROTOCOL", "SASL_PLAINTEXT") - TestUtils.cleanUpCluster() + environmentVariables.set("SCHEMA_REGISTRY_URL", "http://localhost:8082") + Path resourceDirectory = Paths.get("src","test","resources", "plans", "schema_registry", "schemas"); + String absolutePath = resourceDirectory.toFile().getAbsolutePath(); + environmentVariables.set("SCHEMA_DIRECTORY", absolutePath) + TestUtils.cleanUpAll() } void cleanupSpec() { -// TestUtils.cleanUpCluster() +// TestUtils.cleanUpAll() } void 'test various successful applies - #planFile'() { @@ -58,7 +64,8 @@ class ApplyCommandIntegrationSpec extends Specification { "custom-group-id-connect", "custom-application-id-streams", "custom-storage-topic", - "custom-storage-topics" + "custom-storage-topics", + "schema-registry-simple" ] } @@ -89,7 +96,7 @@ class ApplyCommandIntegrationSpec extends Specification { void 'test various valid applies with seed - #planFile #deleteDisabled'() { setup: - TestUtils.seedCluster() + TestUtils.seedKafkaCluster() ByteArrayOutputStream out = new ByteArrayOutputStream() PrintStream oldOut = System.out System.setOut(new PrintStream(out)) @@ -170,4 +177,70 @@ class ApplyCommandIntegrationSpec extends Specification { System.setOut(oldOut) } + void 'test various successful applies schemas - #planFile'() { + setup: + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planFile}-plan.json") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "apply", "-p", file) + + then: + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planFile}-apply-output.txt") + exitCode == 0 + + cleanup: + System.setOut(oldOut) + + where: + planFile << [ + "schema-registry-new-json", + "schema-registry-new-avro", + "schema-registry-new-proto", + "schema-registry-default", + "schema-registry-mix" + ] + } + + void 'test various valid schema registry applies with seed - #planFile #deleteDisabled'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planFile}-plan.json") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = -1 + if (deleteDisabled) { + exitCode = cmd.execute("-f", file, "--no-delete", "apply", "-p", file) + } else { + exitCode = cmd.execute("-f", file, "apply", "-p", file) + } + + then: + if (deleteDisabled) { + assert out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planFile}-no-delete-apply-output.txt") + } else { + assert out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planFile}-apply-output.txt") + } + exitCode == 0 + + cleanup: + System.setOut(oldOut) + + where: + planFile | deleteDisabled + "seed-schema-modification" | false + "seed-schema-modification-2" | false + "seed-schema-modification-3" | false + "seed-schema-modification-4" | false + "seed-schema-modification" | true + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy index c2c0ba23..34ebc759 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy @@ -1,5 +1,7 @@ package com.devshawn.kafka.gitops +import java.nio.file.Path +import java.nio.file.Paths import org.junit.ClassRule import org.junit.contrib.java.lang.system.EnvironmentVariables import org.skyscreamer.jsonassert.JSONAssert @@ -16,16 +18,20 @@ class PlanCommandIntegrationSpec extends Specification { EnvironmentVariables environmentVariables void setupSpec() { - environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") + environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092,localhost:9093,localhost:9094") environmentVariables.set("KAFKA_SASL_JAAS_USERNAME", "test") environmentVariables.set("KAFKA_SASL_JAAS_PASSWORD", "test-secret") environmentVariables.set("KAFKA_SASL_MECHANISM", "PLAIN") environmentVariables.set("KAFKA_SECURITY_PROTOCOL", "SASL_PLAINTEXT") - TestUtils.cleanUpCluster() + environmentVariables.set("SCHEMA_REGISTRY_URL", "http://localhost:8082") + Path resourceDirectory = Paths.get("src","test","resources", "plans", "schema_registry", "schemas"); + String absolutePath = resourceDirectory.toFile().getAbsolutePath(); + environmentVariables.set("SCHEMA_DIRECTORY", absolutePath) + TestUtils.cleanUpAll() } void cleanupSpec() { -// TestUtils.cleanUpCluster() +// TestUtils.cleanUpAll() } void 'test various valid plans - #planName'() { @@ -99,8 +105,7 @@ class PlanCommandIntegrationSpec extends Specification { void 'test various valid plans with seed - #planName'() { setup: - TestUtils.cleanUpCluster() - TestUtils.seedCluster() + TestUtils.seedKafkaCluster() String planOutputFile = "/tmp/plan.json" String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") MainCommand mainCommand = new MainCommand() @@ -139,8 +144,7 @@ class PlanCommandIntegrationSpec extends Specification { void 'test include unchanged flag - #planNam #includeUnchanged'() { setup: - TestUtils.cleanUpCluster() - TestUtils.seedCluster() + TestUtils.seedKafkaCluster() String planOutputFile = "/tmp/plan.json" String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") MainCommand mainCommand = new MainCommand() @@ -256,8 +260,7 @@ class PlanCommandIntegrationSpec extends Specification { void 'test plan that has no changes - #includeUnchanged'() { setup: - TestUtils.cleanUpCluster() - TestUtils.seedCluster() + TestUtils.seedKafkaCluster() ByteArrayOutputStream out = new ByteArrayOutputStream() PrintStream oldOut = System.out System.setOut(new PrintStream(out)) @@ -294,4 +297,208 @@ class PlanCommandIntegrationSpec extends Specification { "no-changes" | false "no-changes" | true } + + void 'test various valid schema registry plans - #planName'() { + setup: + TestUtils.cleanUpAll() + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + + then: + exitCode == 0 + + when: + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") + + then: + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + where: + planName << [ + "schema-registry-new-json", + "schema-registry-new-avro", + "schema-registry-new-proto", + "schema-registry-default", + "schema-registry-mix" + ] + } + + void 'test various valid schema registry plans with seed - #planName'() { + setup: + TestUtils.seedSchemaRegistry() + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode + if (deleteDisabled) { + exitCode = cmd.execute("-f", file, "--no-delete", "plan", "-o", planOutputFile) + } else { + exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + } + + then: + exitCode == 0 + + when: + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") + + then: + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + where: + planName | deleteDisabled + "seed-schema-modification" | false + "seed-schema-modification-2" | false + "seed-schema-modification-3" | false + "seed-schema-modification-4" | false + "seed-schema-modification-no-delete" | true + } + + void 'test schema registry plan that has no changes - #includeUnchanged'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = -1 + if (includeUnchanged) { + exitCode = cmd.execute("-f", file, "plan", "--include-unchanged", "-o", planOutputFile) + } else { + exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + } + + then: + exitCode == 0 + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/no-changes-output.txt") + + when: + String expected = includeUnchanged ? "${planName}-include-unchanged" : planName + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${expected}-plan.json") + + then: + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + cleanup: + System.setOut(oldOut) + + where: + planName | includeUnchanged + "no-changes" | false + "no-changes" | true + } + + void 'test invalid schema registry plans - #planName'() { + setup: + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + + then: + exitCode == 2 + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planName}-output.txt") + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + + where: + planName << [ + "invalid-type", + "invalid-missing-type", + "invalid-compatibility", + "invalid-missing-compatibility", + "invalid-unrecognized-property", + "invalid-missing-file-and-schema", + "invalid-both-file-and-schema" + ] + } + + void 'test various invalid schema registry plans with seed - #planName'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + + then: + exitCode == 2 + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/${planName}-output.txt") + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + + where: + planName << [ + "invalid-modify-type", + "invalid-modify-compatibility", + "invalid-modify-not-compatible" + ] + } + + void 'test various invalid schema registry plans with seed (regex) - #planName'() { + setup: + TestUtils.seedSchemaRegistry() + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + String pattern = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-output.txt") + + then: + exitCode == 2 + out.toString().matches(pattern) + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + + where: + planName << [ + "invalid-reference" + ] + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy index bfc943a1..892e33ed 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy @@ -1,5 +1,6 @@ package com.devshawn.kafka.gitops +import java.nio.file.Paths import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.NewTopic @@ -9,12 +10,29 @@ import org.apache.kafka.common.resource.PatternType import org.apache.kafka.common.resource.ResourcePattern import org.apache.kafka.common.resource.ResourcePatternFilter import org.apache.kafka.common.resource.ResourceType +import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader +import com.devshawn.kafka.gitops.enums.SchemaCompatibility +import com.devshawn.kafka.gitops.enums.SchemaType +import com.devshawn.kafka.gitops.exception.ValidationException +import com.devshawn.kafka.gitops.service.SchemaRegistryService +import groovy.swing.factory.CollectionFactory +import io.confluent.kafka.schemaregistry.AbstractSchemaProvider +import io.confluent.kafka.schemaregistry.ParsedSchema +import io.confluent.kafka.schemaregistry.SchemaProvider +import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.rest.RestService +import io.confluent.kafka.schemaregistry.client.security.basicauth.SaslBasicAuthCredentialProvider +import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider import spock.util.concurrent.PollingConditions -import java.nio.file.Paths - class TestUtils { + private TestUtils() { + } + static String getFileContent(String fileName) { File file = new File(fileName) return file.text @@ -32,17 +50,61 @@ class TestUtils { return file.getAbsolutePath() } - static void cleanUpCluster() { - def conditions = new PollingConditions(timeout: 60, initialDelay: 2, factor: 1.25) + static void cleanUpSchemaRegistry() { + try { + CachedSchemaRegistryClient schemaRegistryClient = getSchemaRegistryClient(); + + Collection subjects = schemaRegistryClient.getAllSubjects(); + for (subject in subjects) { + schemaRegistryClient.deleteSubject(subject); + schemaRegistryClient.deleteSubject(subject, true); + } + assert schemaRegistryClient.getAllSubjects().size() == 0 + println "Finished cleaning up schema registry" + } catch (Exception ex) { + println "Error cleaning up schema registry" + throw new RuntimeException("Error cleaning up schema registry", ex) + } + } + + static void seedSchemaRegistry() { + try { + CachedSchemaRegistryClient schemaRegistryClient = getSchemaRegistryClient(); + createSchema("schema-1-json", SchemaType.JSON, + "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}", schemaRegistryClient, SchemaCompatibility.BACKWARD) + createSchema("schema-2-avro", SchemaType.AVRO, + "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + schemaRegistryClient, SchemaCompatibility.BACKWARD) + createSchema("schema-3-protobuf", SchemaType.PROTOBUF, + "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", + schemaRegistryClient, SchemaCompatibility.FULL) + + println "Finished seeding schema registry" + } catch (Exception ex) { + println "Error seeding up Schema registry" + throw new RuntimeException("Error seeding up schema registry", ex) + } + } + + static void cleanUpAll() { + cleanUpKafkaCluster(); + cleanUpSchemaRegistry() + } + + static void cleanUpKafkaCluster() { + def conditions = new PollingConditions(timeout: 120, initialDelay: 2, factor: 1.25) try { AdminClient adminClient = AdminClient.create(getKafkaConfig()) - Set topics = adminClient.listTopics().names().get() + Set topics = adminClient.listTopics().names().get(); + // Do not remove the schema registry topic + topics.remove("_schemas"); adminClient.deleteTopics(topics).all().get() conditions.eventually { Set remainingTopics = adminClient.listTopics().names().get() - assert remainingTopics.size() == 0 + assert remainingTopics.size() == 1 + assert remainingTopics.getAt(0).equals("_schemas") } AclBindingFilter filter = getWildcardFilter() @@ -51,15 +113,15 @@ class TestUtils { List acls = new ArrayList<>(adminClient.describeAcls(filter).values().get()) assert acls.size() == 0 } - println "Finished cleaning up cluster" + println "Finished cleaning up kafka cluster" } catch (Exception ex) { println "Error cleaning up kafka cluster" - println ex + throw new RuntimeException("Error cleaning up kafka cluster", ex) } } - static void seedCluster() { + static void seedKafkaCluster() { def conditions = new PollingConditions(timeout: 60, initialDelay: 2, factor: 1.25) try { @@ -72,7 +134,7 @@ class TestUtils { conditions.eventually { Set newTopics = adminClient.listTopics().names().get() - assert newTopics.size() == 4 + assert newTopics.size() == 5 List newAcls = new ArrayList<>(adminClient.describeAcls(getWildcardFilter()).values().get()) assert newAcls.size() == 1 @@ -80,10 +142,23 @@ class TestUtils { println "Finished seeding kafka cluster" } catch (Exception ex) { println "Error seeding up kafka cluster" - ex.printStackTrace() + throw new RuntimeException("Error seeding up kafka cluster", ex) } } + static void createSchema(String subject, SchemaType type, String schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { + AbstractSchemaProvider schemaProvider = SchemaRegistryService.schemaProviderFromType(type); + ParsedSchema parsedSchema = schemaProvider.parseSchema(schema, Collections.emptyList()).get(); + createSchema(subject, type, parsedSchema, client, compatibility) + } + + static void createSchema(String subject, SchemaType type, ParsedSchema schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { + CachedSchemaRegistryClient schemaRegistryClient = getSchemaRegistryClient(); + int id = schemaRegistryClient.register(subject, schema); + String compat = schemaRegistryClient.updateCompatibility(subject, compatibility.toString()); + println "Schema subject '" + subject + "' with id " + id + " created (compatibility: " + compat + ")" + } + static void createTopic(String name, int partitions, AdminClient adminClient) { createTopic(name, partitions, adminClient, null) } @@ -119,4 +194,18 @@ class TestUtils { (SaslConfigs.SASL_JAAS_CONFIG) : jaasConfig, ] } + + static CachedSchemaRegistryClient getSchemaRegistryClient() { + Map config = SchemaRegistryConfigLoader.load().getConfig(); + RestService restService = new RestService(config.get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()) + if(config.get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY) != null) { + SaslBasicAuthCredentialProvider saslBasicAuthCredentialProvider = new SaslBasicAuthCredentialProvider() + Map clientConfig = new HashMap<>() + clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, config + .get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY).toString()) + saslBasicAuthCredentialProvider.configure(clientConfig) + restService.setBasicAuthCredentialProvider(saslBasicAuthCredentialProvider) + } + return new CachedSchemaRegistryClient(restService, 10); + } } diff --git a/src/test/resources/plans/application-service-plan.json b/src/test/resources/plans/application-service-plan.json index 72daf4d2..7c87d536 100644 --- a/src/test/resources/plans/application-service-plan.json +++ b/src/test/resources/plans/application-service-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/application-service.yaml b/src/test/resources/plans/application-service.yaml index f9615d40..6367d57e 100644 --- a/src/test/resources/plans/application-service.yaml +++ b/src/test/resources/plans/application-service.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-service: type: application diff --git a/src/test/resources/plans/custom-application-id-streams-plan.json b/src/test/resources/plans/custom-application-id-streams-plan.json index 839e0622..f313955f 100644 --- a/src/test/resources/plans/custom-application-id-streams-plan.json +++ b/src/test/resources/plans/custom-application-id-streams-plan.json @@ -1,187 +1,188 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "streams-application-0", - "aclDetails": { - "name": "my-other-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-0", + "aclDetails" : { + "name" : "my-other-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-1", - "aclDetails": { - "name": "my-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-1", + "aclDetails" : { + "name" : "my-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-2", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-2", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-3", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-3", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-4", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-4", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-5", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-5", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-6", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" + "name" : "streams-application-6", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-7", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" + "name" : "streams-application-7", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-8", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-8", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-9", - "aclDetails": { - "name": "test-streams", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-9", + "aclDetails" : { + "name" : "test-streams", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-10", - "aclDetails": { - "name": "test-streams", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-10", + "aclDetails" : { + "name" : "test-streams", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-11", - "aclDetails": { - "name": "test-streams", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-11", + "aclDetails" : { + "name" : "test-streams", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-12", - "aclDetails": { - "name": "test-streams", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-12", + "aclDetails" : { + "name" : "test-streams", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-13", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-13", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-application-id-streams.yaml b/src/test/resources/plans/custom-application-id-streams.yaml index 57cc60e5..5ff9b2a4 100644 --- a/src/test/resources/plans/custom-application-id-streams.yaml +++ b/src/test/resources/plans/custom-application-id-streams.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: streams-application: type: kafka-streams diff --git a/src/test/resources/plans/custom-group-id-application-plan.json b/src/test/resources/plans/custom-group-id-application-plan.json index 8db50dae..c3a9b966 100644 --- a/src/test/resources/plans/custom-group-id-application-plan.json +++ b/src/test/resources/plans/custom-group-id-application-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-group-id-application.yaml b/src/test/resources/plans/custom-group-id-application.yaml index f3be06cc..3e52fa18 100644 --- a/src/test/resources/plans/custom-group-id-application.yaml +++ b/src/test/resources/plans/custom-group-id-application.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-service: type: application diff --git a/src/test/resources/plans/custom-group-id-connect-plan.json b/src/test/resources/plans/custom-group-id-connect-plan.json index 579b8078..f97a438d 100644 --- a/src/test/resources/plans/custom-group-id-connect-plan.json +++ b/src/test/resources/plans/custom-group-id-connect-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-group-id-connect.yaml b/src/test/resources/plans/custom-group-id-connect.yaml index 11e6e57e..c5bea388 100644 --- a/src/test/resources/plans/custom-group-id-connect.yaml +++ b/src/test/resources/plans/custom-group-id-connect.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/custom-service-acls-plan.json b/src/test/resources/plans/custom-service-acls-plan.json index d20095ea..1112b9f3 100644 --- a/src/test/resources/plans/custom-service-acls-plan.json +++ b/src/test/resources/plans/custom-service-acls-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "kafka.", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "kafka.", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-service-acls.yaml b/src/test/resources/plans/custom-service-acls.yaml index 31e4a26e..6112627c 100644 --- a/src/test/resources/plans/custom-service-acls.yaml +++ b/src/test/resources/plans/custom-service-acls.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-service: type: application diff --git a/src/test/resources/plans/custom-storage-topic-plan.json b/src/test/resources/plans/custom-storage-topic-plan.json index fcdc8f97..8b87ee09 100644 --- a/src/test/resources/plans/custom-storage-topic-plan.json +++ b/src/test/resources/plans/custom-storage-topic-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "test-connect-cluster-configs", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "test-connect-cluster-configs", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "test-connect-cluster-configs", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "test-connect-cluster-configs", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "test-connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "test-connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-storage-topic.yaml b/src/test/resources/plans/custom-storage-topic.yaml index 57df46fc..3c7b5d04 100644 --- a/src/test/resources/plans/custom-storage-topic.yaml +++ b/src/test/resources/plans/custom-storage-topic.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/custom-storage-topics-plan.json b/src/test/resources/plans/custom-storage-topics-plan.json index 5d25fadf..e4d78f5b 100644 --- a/src/test/resources/plans/custom-storage-topics-plan.json +++ b/src/test/resources/plans/custom-storage-topics-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "config-custom-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "config-custom-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "offset-topic-custom", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "offset-topic-custom", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "custom-status-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "custom-status-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "config-custom-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "config-custom-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "offset-topic-custom", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "offset-topic-custom", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "custom-status-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "custom-status-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "test-connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "test-connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-storage-topics.yaml b/src/test/resources/plans/custom-storage-topics.yaml index 0a56bd64..f799e2e9 100644 --- a/src/test/resources/plans/custom-storage-topics.yaml +++ b/src/test/resources/plans/custom-storage-topics.yaml @@ -1,3 +1,8 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/custom-user-acls-plan.json b/src/test/resources/plans/custom-user-acls-plan.json index 9c8eba5c..a0af374e 100644 --- a/src/test/resources/plans/custom-user-acls-plan.json +++ b/src/test/resources/plans/custom-user-acls-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-user-0", - "aclDetails": { - "name": "kafka.", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-0", + "aclDetails" : { + "name" : "kafka.", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/custom-user-acls.yaml b/src/test/resources/plans/custom-user-acls.yaml index caf671ab..2fd46d64 100644 --- a/src/test/resources/plans/custom-user-acls.yaml +++ b/src/test/resources/plans/custom-user-acls.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + users: test-user: principal: User:test diff --git a/src/test/resources/plans/default-replication-multiple-plan.json b/src/test/resources/plans/default-replication-multiple-plan.json index 5c816376..8fbd5389 100644 --- a/src/test/resources/plans/default-replication-multiple-plan.json +++ b/src/test/resources/plans/default-replication-multiple-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 3, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 3, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "last-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 4, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "last-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 4, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/default-replication-multiple.yaml b/src/test/resources/plans/default-replication-multiple.yaml index 05aee33d..08192a03 100644 --- a/src/test/resources/plans/default-replication-multiple.yaml +++ b/src/test/resources/plans/default-replication-multiple.yaml @@ -2,7 +2,9 @@ settings: topics: defaults: replication: 3 - + blacklist: + prefixed: + - _schemas topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/default-replication-plan.json b/src/test/resources/plans/default-replication-plan.json index e841e1a1..03744be7 100644 --- a/src/test/resources/plans/default-replication-plan.json +++ b/src/test/resources/plans/default-replication-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/default-replication.yaml b/src/test/resources/plans/default-replication.yaml index fc5932b9..09f48932 100644 --- a/src/test/resources/plans/default-replication.yaml +++ b/src/test/resources/plans/default-replication.yaml @@ -2,7 +2,9 @@ settings: topics: defaults: replication: 2 - + blacklist: + prefixed: + - _schemas topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/describe-topic-acl-disabled-plan.json b/src/test/resources/plans/describe-topic-acl-disabled-plan.json index 130ee3cc..4972b923 100644 --- a/src/test/resources/plans/describe-topic-acl-disabled-plan.json +++ b/src/test/resources/plans/describe-topic-acl-disabled-plan.json @@ -1,369 +1,370 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "normal-application-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "normal-application-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "normal-application-1", - "aclDetails": { - "name": "first-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "normal-application-1", + "aclDetails" : { + "name" : "first-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "normal-application-2", - "aclDetails": { - "name": "normal-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "normal-application-2", + "aclDetails" : { + "name" : "normal-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-0", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-0", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-1", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-1", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-2", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-2", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-3", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-3", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-4", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-4", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-5", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-5", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-6", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" + "name" : "streams-application-6", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-7", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" + "name" : "streams-application-7", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-8", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-8", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-9", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-9", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-10", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-10", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-11", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-11", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-12", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-12", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-13", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-13", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-0", - "aclDetails": { - "name": "poison-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-0", + "aclDetails" : { + "name" : "poison-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-1", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-1", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-2", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-2", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-3", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-3", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-4", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-4", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-5", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-5", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-6", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-6", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-7", - "aclDetails": { - "name": "kafka-connect-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-7", + "aclDetails" : { + "name" : "kafka-connect-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-8", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "kafka-connect-application-8", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-9", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-9", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "kafka-connect-application-10", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "kafka-connect-application-10", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/describe-topic-acl-disabled.yaml b/src/test/resources/plans/describe-topic-acl-disabled.yaml index c93e625a..2adf1d8d 100644 --- a/src/test/resources/plans/describe-topic-acl-disabled.yaml +++ b/src/test/resources/plans/describe-topic-acl-disabled.yaml @@ -2,7 +2,10 @@ settings: services: acls: describeTopicEnabled: false - + topics: + blacklist: + prefixed: + - _schemas services: normal-application: type: application diff --git a/src/test/resources/plans/describe-topic-acl-enabled-plan.json b/src/test/resources/plans/describe-topic-acl-enabled-plan.json index 689b778c..724e7c10 100644 --- a/src/test/resources/plans/describe-topic-acl-enabled-plan.json +++ b/src/test/resources/plans/describe-topic-acl-enabled-plan.json @@ -1,460 +1,461 @@ { - "topicPlans": [], - "aclPlans": [ - { - "name": "normal-application-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-1", - "aclDetails": { - "name": "first-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-2", - "aclDetails": { - "name": "first-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-3", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "normal-application-4", - "aclDetails": { - "name": "normal-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-0", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-1", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-2", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-3", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-4", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-5", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-6", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-7", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-8", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-9", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-10", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-11", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-12", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-13", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-14", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "streams-application-15", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-0", - "aclDetails": { - "name": "poison-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-1", - "aclDetails": { - "name": "poison-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-2", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-3", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-4", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-5", - "aclDetails": { - "name": "connect-configs-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-6", - "aclDetails": { - "name": "connect-offsets-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-7", - "aclDetails": { - "name": "connect-status-kafka-connect-application", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-8", - "aclDetails": { - "name": "kafka-connect-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-9", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-10", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-11", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-12", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" - }, - "action": "ADD" - }, - { - "name": "kafka-connect-application-13", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" - }, - "action": "ADD" + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ + { + "name" : "normal-application-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-1", + "aclDetails" : { + "name" : "first-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-2", + "aclDetails" : { + "name" : "first-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-3", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "normal-application-4", + "aclDetails" : { + "name" : "normal-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-0", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-1", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-2", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-3", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-4", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-5", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-6", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-7", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-8", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-9", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-10", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-11", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-12", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-13", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-14", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "streams-application-15", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-0", + "aclDetails" : { + "name" : "poison-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-1", + "aclDetails" : { + "name" : "poison-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-2", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-3", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-4", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-5", + "aclDetails" : { + "name" : "connect-configs-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-6", + "aclDetails" : { + "name" : "connect-offsets-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-7", + "aclDetails" : { + "name" : "connect-status-kafka-connect-application", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-8", + "aclDetails" : { + "name" : "kafka-connect-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-9", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-10", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-11", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-12", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "kafka-connect-application-13", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/describe-topic-acl-enabled.yaml b/src/test/resources/plans/describe-topic-acl-enabled.yaml index d19d782d..27a50cf0 100644 --- a/src/test/resources/plans/describe-topic-acl-enabled.yaml +++ b/src/test/resources/plans/describe-topic-acl-enabled.yaml @@ -2,7 +2,10 @@ settings: services: acls: describeTopicEnabled: true - + topics: + blacklist: + prefixed: + - _schemas services: normal-application: type: application diff --git a/src/test/resources/plans/invalid-custom-user-acls-1.yaml b/src/test/resources/plans/invalid-custom-user-acls-1.yaml index 9aa74b00..1382f6c0 100644 --- a/src/test/resources/plans/invalid-custom-user-acls-1.yaml +++ b/src/test/resources/plans/invalid-custom-user-acls-1.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + users: test-user: principal: User:test diff --git a/src/test/resources/plans/invalid-plan.json b/src/test/resources/plans/invalid-plan.json index 96833b5b..77685ada 100644 --- a/src/test/resources/plans/invalid-plan.json +++ b/src/test/resources/plans/invalid-plan.json @@ -1,3 +1,3 @@ { - "topicPlans": {} + "topicPlans" : {} } \ No newline at end of file diff --git a/src/test/resources/plans/invalid-storage-topics.yaml b/src/test/resources/plans/invalid-storage-topics.yaml index 9f2833a6..8d42e298 100644 --- a/src/test/resources/plans/invalid-storage-topics.yaml +++ b/src/test/resources/plans/invalid-storage-topics.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/invalid-topic-remove-partitions.yaml b/src/test/resources/plans/invalid-topic-remove-partitions.yaml index c5af5b16..b0214936 100644 --- a/src/test/resources/plans/invalid-topic-remove-partitions.yaml +++ b/src/test/resources/plans/invalid-topic-remove-partitions.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/invalid-topic.yaml b/src/test/resources/plans/invalid-topic.yaml index 82e7e7e7..22d538fd 100644 --- a/src/test/resources/plans/invalid-topic.yaml +++ b/src/test/resources/plans/invalid-topic.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: replication: 2 \ No newline at end of file diff --git a/src/test/resources/plans/kafka-connect-service-plan.json b/src/test/resources/plans/kafka-connect-service-plan.json index 14d019fe..50068b8e 100644 --- a/src/test/resources/plans/kafka-connect-service-plan.json +++ b/src/test/resources/plans/kafka-connect-service-plan.json @@ -1,135 +1,136 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-connect-cluster-0", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-0", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-1", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-1", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-2", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-2", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-3", - "aclDetails": { - "name": "connect-configs-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-3", + "aclDetails" : { + "name" : "connect-configs-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-4", - "aclDetails": { - "name": "connect-offsets-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-4", + "aclDetails" : { + "name" : "connect-offsets-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-5", - "aclDetails": { - "name": "connect-status-test-connect-cluster", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-5", + "aclDetails" : { + "name" : "connect-status-test-connect-cluster", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-6", - "aclDetails": { - "name": "test-connect-cluster", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-6", + "aclDetails" : { + "name" : "test-connect-cluster", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-7", - "aclDetails": { - "name": "production-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-connect-cluster-7", + "aclDetails" : { + "name" : "production-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-8", - "aclDetails": { - "name": "consumption-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-8", + "aclDetails" : { + "name" : "consumption-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-connect-cluster-9", - "aclDetails": { - "name": "connect-test-sink", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-connect-cluster-9", + "aclDetails" : { + "name" : "connect-test-sink", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/kafka-connect-service.yaml b/src/test/resources/plans/kafka-connect-service.yaml index c7690046..b38613f4 100644 --- a/src/test/resources/plans/kafka-connect-service.yaml +++ b/src/test/resources/plans/kafka-connect-service.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: test-connect-cluster: type: kafka-connect diff --git a/src/test/resources/plans/kafka-streams-service-plan.json b/src/test/resources/plans/kafka-streams-service-plan.json index c752cd52..d2145b9a 100644 --- a/src/test/resources/plans/kafka-streams-service-plan.json +++ b/src/test/resources/plans/kafka-streams-service-plan.json @@ -1,200 +1,201 @@ { - "topicPlans": [], - "aclPlans": [ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "streams-application-0", - "aclDetails": { - "name": "my-other-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-0", + "aclDetails" : { + "name" : "my-other-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-1", - "aclDetails": { - "name": "another-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-1", + "aclDetails" : { + "name" : "another-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-2", - "aclDetails": { - "name": "my-streams-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-2", + "aclDetails" : { + "name" : "my-streams-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-3", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-3", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-4", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "streams-application-4", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-5", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-5", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-6", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-6", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-7", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "CREATE", - "permission": "ALLOW" + "name" : "streams-application-7", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "CREATE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-8", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER", - "permission": "ALLOW" + "name" : "streams-application-8", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-9", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "ALTER_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-9", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "ALTER_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-10", - "aclDetails": { - "name": "streams-application", - "type": "TOPIC", - "pattern": "PREFIXED", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-10", + "aclDetails" : { + "name" : "streams-application", + "type" : "TOPIC", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-11", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "streams-application-11", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-12", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "streams-application-12", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-13", - "aclDetails": { - "name": "streams-application", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DELETE", - "permission": "ALLOW" + "name" : "streams-application-13", + "aclDetails" : { + "name" : "streams-application", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DELETE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "streams-application-14", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "streams-application-14", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/kafka-streams-service.yaml b/src/test/resources/plans/kafka-streams-service.yaml index 3c408bb0..11fc9787 100644 --- a/src/test/resources/plans/kafka-streams-service.yaml +++ b/src/test/resources/plans/kafka-streams-service.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + services: streams-application: type: kafka-streams diff --git a/src/test/resources/plans/multi-file-plan.json b/src/test/resources/plans/multi-file-plan.json index 0a294973..2a654f18 100644 --- a/src/test/resources/plans/multi-file-plan.json +++ b/src/test/resources/plans/multi-file-plan.json @@ -1,58 +1,59 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "another-test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/multi-file.yaml b/src/test/resources/plans/multi-file.yaml index 62c6b1ea..9ff52700 100644 --- a/src/test/resources/plans/multi-file.yaml +++ b/src/test/resources/plans/multi-file.yaml @@ -3,3 +3,7 @@ settings: services: multi-file-services.yaml topics: multi-file-topics.yaml users: multi-file-users.yaml + topics: + blacklist: + prefixed: + - _schemas diff --git a/src/test/resources/plans/no-changes-include-unchanged-plan.json b/src/test/resources/plans/no-changes-include-unchanged-plan.json index d3bd7798..f667e7a8 100644 --- a/src/test/resources/plans/no-changes-include-unchanged-plan.json +++ b/src/test/resources/plans/no-changes-include-unchanged-plan.json @@ -1,91 +1,92 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "delete-topic", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "delete-topic", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "test-topic", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "test-topic", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-1", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "NO_CHANGE" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "NO_CHANGE" } ] }, { - "name": "topic-with-configs-2", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-2", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "NO_CHANGE" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "NO_CHANGE" + "action" : "NO_CHANGE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/no-changes-plan.json b/src/test/resources/plans/no-changes-plan.json index b93dada0..6dbd87bc 100644 --- a/src/test/resources/plans/no-changes-plan.json +++ b/src/test/resources/plans/no-changes-plan.json @@ -1,4 +1,5 @@ { - "topicPlans": [], - "aclPlans": [] + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/no-changes.yaml b/src/test/resources/plans/no-changes.yaml index c4b2de14..56f672d0 100644 --- a/src/test/resources/plans/no-changes.yaml +++ b/src/test/resources/plans/no-changes.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/schema_registry/add-with-reference.yaml b/src/test/resources/plans/schema_registry/add-with-reference.yaml new file mode 100644 index 00000000..bb261662 --- /dev/null +++ b/src/test/resources/plans/schema_registry/add-with-reference.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-2-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: 1 \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-both-file-and-schema-output.txt b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema-output.txt new file mode 100644 index 00000000..215c520d --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] schema and file fields cannot be both set at the same time in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-both-file-and-schema.yaml b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema.yaml new file mode 100644 index 00000000..60134094 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-both-file-and-schema.yaml @@ -0,0 +1,12 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: NONE + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + file: "schema-registry-schema1.json" diff --git a/src/test/resources/plans/schema_registry/invalid-compatibility-output.txt b/src/test/resources/plans/schema_registry/invalid-compatibility-output.txt new file mode 100644 index 00000000..057c9a6b --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-compatibility-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Value 'XXX' is not a valid format for: [compatibility] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-compatibility.yaml b/src/test/resources/plans/schema_registry/invalid-compatibility.yaml new file mode 100644 index 00000000..a5fadab2 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-compatibility.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: XXX + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-missing-compatibility-output.txt b/src/test/resources/plans/schema_registry/invalid-missing-compatibility-output.txt new file mode 100644 index 00000000..50175cdb --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-compatibility-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Not set: [compatibility] in state file definition: schema -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-missing-compatibility.yaml b/src/test/resources/plans/schema_registry/invalid-missing-compatibility.yaml new file mode 100644 index 00000000..3aa63b19 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-compatibility.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema-output.txt b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema-output.txt new file mode 100644 index 00000000..6b6d8665 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] schema or file field must be provided in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema.yaml b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema.yaml new file mode 100644 index 00000000..46d1dbd1 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-file-and-schema.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt b/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt new file mode 100644 index 00000000..0996def9 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] type not set in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-missing-type.yaml b/src/test/resources/plans/schema_registry/invalid-missing-type.yaml new file mode 100644 index 00000000..c9c952fc --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-missing-type.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-modify-compatibility-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-compatibility-output.txt new file mode 100644 index 00000000..5b11b4af --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-compatibility-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Changing the subject compatibility is not allowed with kafka-gitops(subject: schema-1-json, current compatibilty: BACKWARD, new compatibilty:FORWARD) diff --git a/src/test/resources/plans/schema_registry/invalid-modify-compatibility.yaml b/src/test/resources/plans/schema_registry/invalid-modify-compatibility.yaml new file mode 100644 index 00000000..179d921c --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-compatibility.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt new file mode 100644 index 00000000..07a01874 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] JSON schema 'schema-1-json' is not compatible with the latest one: [Found incompatible change: Difference{jsonPath='#/properties/f1', type=PROPERTY_REMOVED_FROM_CLOSED_CONTENT_MODEL}] diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible.yaml b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible.yaml new file mode 100644 index 00000000..d99d0a99 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f2\":{\"type\":\"string\"}}, \"additionalProperties\": false}" \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt new file mode 100644 index 00000000..0bcf228f --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] AVRO schema 'schema-1-json' is not compatible with the latest one: [Incompatible because of different schema type] diff --git a/src/test/resources/plans/schema_registry/invalid-modify-type.yaml b/src/test/resources/plans/schema_registry/invalid-modify-type.yaml new file mode 100644 index 00000000..2544c8fb --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-type.yaml @@ -0,0 +1,24 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-2-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-1-json: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +}' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-reference-output.txt b/src/test/resources/plans/schema_registry/invalid-reference-output.txt new file mode 100644 index 00000000..9372f7f1 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-reference-output.txt @@ -0,0 +1 @@ +(?s).*Version 10 not found\..*\[INVALID\] JSON referenced schema could not be parsed for subject schema-2-json diff --git a/src/test/resources/plans/schema_registry/invalid-reference.yaml b/src/test/resources/plans/schema_registry/invalid-reference.yaml new file mode 100644 index 00000000..95d0180a --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-reference.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-2-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: 10 \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-type-output.txt b/src/test/resources/plans/schema_registry/invalid-type-output.txt new file mode 100644 index 00000000..cb65ae27 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-type-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Value 'XXXX' is not a valid format for: [type] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-type.yaml b/src/test/resources/plans/schema_registry/invalid-type.yaml new file mode 100644 index 00000000..f4ff1467 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-type.yaml @@ -0,0 +1,10 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: XXXX + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" diff --git a/src/test/resources/plans/schema_registry/invalid-unrecognized-property-output.txt b/src/test/resources/plans/schema_registry/invalid-unrecognized-property-output.txt new file mode 100644 index 00000000..5dceccd2 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-unrecognized-property-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[INVALID] Unrecognized field: [what] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-unrecognized-property.yaml b/src/test/resources/plans/schema_registry/invalid-unrecognized-property.yaml new file mode 100644 index 00000000..d945d85a --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-unrecognized-property.yaml @@ -0,0 +1,12 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + what: "" diff --git a/src/test/resources/plans/schema_registry/no-changes-include-unchanged-plan.json b/src/test/resources/plans/schema_registry/no-changes-include-unchanged-plan.json new file mode 100644 index 00000000..1bd12923 --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes-include-unchanged-plan.json @@ -0,0 +1,39 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-1-json", + "action" : "NO_CHANGE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "schema-2-avro", + "action" : "NO_CHANGE", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "schema-3-protobuf", + "action" : "NO_CHANGE", + "schemaDetails" : { + "type" : "PROTOBUF", + "schema" : "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/no-changes-output.txt b/src/test/resources/plans/schema_registry/no-changes-output.txt new file mode 100644 index 00000000..9b86d9f0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes-output.txt @@ -0,0 +1,3 @@ +Generating execution plan... + +[SUCCESS] There are no necessary changes; the actual state matches the desired state. diff --git a/src/test/resources/plans/schema_registry/no-changes-plan.json b/src/test/resources/plans/schema_registry/no-changes-plan.json new file mode 100644 index 00000000..6dbd87bc --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes-plan.json @@ -0,0 +1,5 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/no-changes.yaml b/src/test/resources/plans/schema_registry/no-changes.yaml new file mode 100644 index 00000000..10fd50f9 --- /dev/null +++ b/src/test/resources/plans/schema_registry/no-changes.yaml @@ -0,0 +1,24 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +}' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt new file mode 100644 index 00000000..cf67b708 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt @@ -0,0 +1,29 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value2 + + type: JSON + + compatibility: FULL + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] json-value3 + + type: JSON + + compatibility: NONE + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 2 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-default-plan.json b/src/test/resources/plans/schema_registry/schema-registry-default-plan.json new file mode 100644 index 00000000..7f302c6a --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-default-plan.json @@ -0,0 +1,28 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value2", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + }, + { + "name" : "json-value3", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-default.yaml b/src/test/resources/plans/schema_registry/schema-registry-default.yaml new file mode 100644 index 00000000..8bb2d667 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-default.yaml @@ -0,0 +1,17 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + schema: + defaults: + compatibility: FULL + +schemas: + json-value2: + type: JSON + file: schema-registry-schema1.json + json-value3: + type: JSON + compatibility: NONE + file: schema-registry-schema1.json \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt new file mode 100644 index 00000000..72b3a16c --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt @@ -0,0 +1,42 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: FULL + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] avro-value1 + + type: AVRO + + compatibility: NONE + + schema: +---------------------- +{"type":"record","name":"TestRecord","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello","type":"string"}]} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] avro-value2 + + type: AVRO + + compatibility: FORWARD + + schema: +---------------------- +{"type":"record","name":"TestRecord2","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello2","type":"string"}]} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 3 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix-plan.json b/src/test/resources/plans/schema_registry/schema-registry-mix-plan.json new file mode 100644 index 00000000..65424f72 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-mix-plan.json @@ -0,0 +1,39 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + }, + { + "name" : "avro-value1", + "action" : "ADD", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + }, + { + "name" : "avro-value2", + "action" : "ADD", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord2\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello2\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix.yaml b/src/test/resources/plans/schema_registry/schema-registry-mix.yaml new file mode 100644 index 00000000..a3a2059e --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-mix.yaml @@ -0,0 +1,31 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + schema: + defaults: + compatibility: FULL + +schemas: + json-value: + type: JSON + file: schema-registry-schema1.json + avro-value1: + type: AVRO + compatibility: NONE + file: schema-registry-schema2.avsc + avro-value2: + type: AVRO + compatibility: FORWARD + schema : '{ + "type": "record", + "name": "TestRecord2", + "namespace": "com.devshawn.kafka.gitops", + "fields": [ + { + "name": "hello2", + "type": "string" + } + ] +}' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt new file mode 100644 index 00000000..a5840d19 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt @@ -0,0 +1,16 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] avro-value + + type: AVRO + + compatibility: FULL + + schema: +---------------------- +{"type":"record","name":"TestRecord","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello","type":"string"}]} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-avro-plan.json b/src/test/resources/plans/schema_registry/schema-registry-new-avro-plan.json new file mode 100644 index 00000000..0bfecfc2 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-avro-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "avro-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "AVRO", + "schema" : "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + "file" : null, + "compatibility" : "FULL", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-avro.yaml b/src/test/resources/plans/schema_registry/schema-registry-new-avro.yaml new file mode 100644 index 00000000..b7170db7 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-avro.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + avro-value: + type: AVRO + compatibility: FULL + file: schema-registry-schema2.avsc \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt new file mode 100644 index 00000000..995325b9 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt @@ -0,0 +1,16 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-json-plan.json b/src/test/resources/plans/schema_registry/schema-registry-new-json-plan.json new file mode 100644 index 00000000..b6e7cd6c --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-json-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-json.yaml b/src/test/resources/plans/schema_registry/schema-registry-new-json.yaml new file mode 100644 index 00000000..606cb35a --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-json.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + json-value: + type: JSON + compatibility: BACKWARD + file: schema-registry-schema1.json \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt new file mode 100644 index 00000000..8038e1aa --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt @@ -0,0 +1,22 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] proto-value + + type: PROTOBUF + + compatibility: NONE + + schema: +---------------------- +syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} + +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-proto-plan.json b/src/test/resources/plans/schema_registry/schema-registry-new-proto-plan.json new file mode 100644 index 00000000..d2bef497 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-proto-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "proto-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "PROTOBUF", + "schema" : "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-proto.yaml b/src/test/resources/plans/schema_registry/schema-registry-new-proto.yaml new file mode 100644 index 00000000..81f151c8 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schema-registry-new-proto.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + proto-value: + type: PROTOBUF + compatibility: NONE + file: schema-registry-schema3.proto \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema1.json b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema1.json new file mode 100644 index 00000000..7e6cfcc0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema1.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "properties": { + "f1": { + "type": "string" + } + } +} diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema2.avsc b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema2.avsc new file mode 100644 index 00000000..8530806d --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema2.avsc @@ -0,0 +1,11 @@ +{ + "type": "record", + "name": "TestRecord", + "namespace": "com.devshawn.kafka.gitops", + "fields": [ + { + "name": "hello", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema3.proto b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema3.proto new file mode 100644 index 00000000..70c11b4b --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema3.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt new file mode 100644 index 00000000..f79340d0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt @@ -0,0 +1,23 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: FORWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [UPDATE] + +~ [SCHEMA] schema-1-json + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 1 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-2-plan.json new file mode 100644 index 00000000..5aab8acf --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2-plan.json @@ -0,0 +1,28 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ ] + } + }, + { + "name" : "schema-1-json", + "action" : "UPDATE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml new file mode 100644 index 00000000..3cfe6317 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml @@ -0,0 +1,28 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + json-value: + type: JSON + compatibility: FORWARD + schema: '{"type": "object","properties": {"f1": {"type": "string"}}}' + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +}' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-3-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-3-apply-output.txt new file mode 100644 index 00000000..e6024998 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-3-apply-output.txt @@ -0,0 +1,17 @@ +Executing apply... + +Applying: [UPDATE] + +~ [SCHEMA] schema-1-json + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 0 created, 1 updated, 1 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-3-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-3-plan.json new file mode 100644 index 00000000..484c3167 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-3-plan.json @@ -0,0 +1,22 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-1-json", + "action" : "UPDATE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-3.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-3.yaml new file mode 100644 index 00000000..85ee3fc0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-3.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt new file mode 100644 index 00000000..2384b028 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt @@ -0,0 +1,20 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] schema-2-json + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"$ref":"otherschema"}},"additionalProperties":false} +---------------------- + + references: + + name: otherschema + + subject: schema-1-json + + version: -1 + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-4-plan.json new file mode 100644 index 00000000..29534810 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4-plan.json @@ -0,0 +1,23 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-2-json", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ + { + "name" : "otherschema", + "subject" : "schema-1-json", + "version" : -1 + } + ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml new file mode 100644 index 00000000..b471023e --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml @@ -0,0 +1,32 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-2-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: -1 + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" + schema-3-protobuf: + type: PROTOBUF + compatibility: FULL + schema: 'syntax = "proto3"; +package com.acme; + +message OtherRecord { + int32 an_id = 1; +}' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt new file mode 100644 index 00000000..38a38ffb --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt @@ -0,0 +1,50 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] json-value1 + + type: JSON + + compatibility: NONE + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-2-avro + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-1-json + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 2 created, 0 updated, 3 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt new file mode 100644 index 00000000..91db1a03 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt @@ -0,0 +1,29 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] json-value + + type: JSON + + compatibility: BACKWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +Applying: [CREATE] + ++ [SCHEMA] json-value1 + + type: JSON + + compatibility: NONE + + schema: +---------------------- +{"type":"object","properties":{"f1":{"type":"string"}}} +---------------------- + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 2 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-plan.json new file mode 100644 index 00000000..14202234 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-plan.json @@ -0,0 +1,17 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-1-json", + "action" : "UPDATE", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete.yaml new file mode 100644 index 00000000..85ee3fc0 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"},\"f2\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-avro: + type: AVRO + compatibility: BACKWARD + schema: "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}" diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-plan.json new file mode 100644 index 00000000..ca6ab83e --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-plan.json @@ -0,0 +1,43 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "json-value1", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + }, + { + "name" : "schema-2-avro", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-1-json", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification.yaml new file mode 100644 index 00000000..c59b92a4 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification.yaml @@ -0,0 +1,15 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + json-value: + type: JSON + compatibility: BACKWARD + schema: '{"type": "object","properties": {"f1": {"type": "string"}}}' + json-value1: + type: JSON + compatibility: NONE + file: schema-registry-schema1.json \ No newline at end of file diff --git a/src/test/resources/plans/seed-acl-exists-plan.json b/src/test/resources/plans/seed-acl-exists-plan.json index a0fb8fb2..6a1fd21a 100644 --- a/src/test/resources/plans/seed-acl-exists-plan.json +++ b/src/test/resources/plans/seed-acl-exists-plan.json @@ -1,57 +1,58 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": null, - "previousValue": "compact", - "action": "REMOVE" + "key" : "cleanup.policy", + "value" : null, + "previousValue" : "compact", + "action" : "REMOVE" }, { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-1", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-acl-exists.yaml b/src/test/resources/plans/seed-acl-exists.yaml index 81a5c7f6..1afa3411 100644 --- a/src/test/resources/plans/seed-acl-exists.yaml +++ b/src/test/resources/plans/seed-acl-exists.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-basic-include-unchanged-plan.json b/src/test/resources/plans/seed-basic-include-unchanged-plan.json index a9672db2..b5ae5978 100644 --- a/src/test/resources/plans/seed-basic-include-unchanged-plan.json +++ b/src/test/resources/plans/seed-basic-include-unchanged-plan.json @@ -1,142 +1,143 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "delete-topic", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "delete-topic", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "NO_CHANGE" }, { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "400000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "400000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-2", - "action": "NO_CHANGE", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 2, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-2", + "action" : "NO_CHANGE", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "NO_CHANGE" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "NO_CHANGE" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "test-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "test-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "NO_CHANGE" + "action" : "NO_CHANGE" }, { - "name": "test-service-1", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-0", - "aclDetails": { - "name": "delete-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-0", + "aclDetails" : { + "name" : "delete-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-1", - "aclDetails": { - "name": "new-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-1", + "aclDetails" : { + "name" : "new-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-basic-plan.json b/src/test/resources/plans/seed-basic-plan.json index 2d8450cd..32c6f86e 100644 --- a/src/test/resources/plans/seed-basic-plan.json +++ b/src/test/resources/plans/seed-basic-plan.json @@ -1,83 +1,84 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "400000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "400000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 3, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 3, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "test-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "test-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-1", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-0", - "aclDetails": { - "name": "delete-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-0", + "aclDetails" : { + "name" : "delete-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "new-service-1", - "aclDetails": { - "name": "new-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:new", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "new-service-1", + "aclDetails" : { + "name" : "new-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:new", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-basic.yaml b/src/test/resources/plans/seed-basic.yaml index d3c3a09b..715a55bc 100644 --- a/src/test/resources/plans/seed-basic.yaml +++ b/src/test/resources/plans/seed-basic.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-blacklist-topics-plan.json b/src/test/resources/plans/seed-blacklist-topics-plan.json index a51fdfa2..b3bc8767 100644 --- a/src/test/resources/plans/seed-blacklist-topics-plan.json +++ b/src/test/resources/plans/seed-blacklist-topics-plan.json @@ -1,50 +1,51 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-2", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-1", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-blacklist-topics.yaml b/src/test/resources/plans/seed-blacklist-topics.yaml index 8b80b3cd..f04cf192 100644 --- a/src/test/resources/plans/seed-blacklist-topics.yaml +++ b/src/test/resources/plans/seed-blacklist-topics.yaml @@ -3,6 +3,7 @@ settings: blacklist: prefixed: - test + - _schemas topics: new-topic: diff --git a/src/test/resources/plans/seed-topic-add-partitions-plan.json b/src/test/resources/plans/seed-topic-add-partitions-plan.json index 9cee3b65..0184e6a6 100644 --- a/src/test/resources/plans/seed-topic-add-partitions-plan.json +++ b/src/test/resources/plans/seed-topic-add-partitions-plan.json @@ -1,45 +1,46 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": 4, - "previousPartitions": 3, - "partitionsAction": "UPDATE", - "replication": null, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : 4, + "previousPartitions" : 3, + "partitionsAction" : "UPDATE", + "replication" : null, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": 10, - "previousPartitions": 6, - "partitionsAction": "UPDATE", - "replication": null, - "previousReplication": null, - "replicationAction": "NO_CHANGE" + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : 10, + "previousPartitions" : 6, + "partitionsAction" : "UPDATE", + "replication" : null, + "previousReplication" : null, + "replicationAction" : "NO_CHANGE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-add-partitions.yaml b/src/test/resources/plans/seed-topic-add-partitions.yaml index 7d93f433..099b56f7 100644 --- a/src/test/resources/plans/seed-topic-add-partitions.yaml +++ b/src/test/resources/plans/seed-topic-add-partitions.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-add-replicas-plan.json b/src/test/resources/plans/seed-topic-add-replicas-plan.json index b194f9fd..99318506 100644 --- a/src/test/resources/plans/seed-topic-add-replicas-plan.json +++ b/src/test/resources/plans/seed-topic-add-replicas-plan.json @@ -1,45 +1,46 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": null, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 3, - "previousReplication": 2, - "replicationAction": "UPDATE" + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : null, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 3, + "previousReplication" : 2, + "replicationAction" : "UPDATE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": { - "partitions": null, - "previousPartitions": null, - "partitionsAction": "NO_CHANGE", - "replication": 3, - "previousReplication": 2, - "replicationAction": "UPDATE" + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : { + "partitions" : null, + "previousPartitions" : null, + "partitionsAction" : "NO_CHANGE", + "replication" : 3, + "previousReplication" : 2, + "replicationAction" : "UPDATE" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-add-replicas.json b/src/test/resources/plans/seed-topic-add-replicas.json index 668f8387..6a4eb537 100644 --- a/src/test/resources/plans/seed-topic-add-replicas.json +++ b/src/test/resources/plans/seed-topic-add-replicas.json @@ -1,83 +1,84 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "segment.bytes", - "value": null, - "previousValue": null, - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : null, + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "100000", - "previousValue": "60000", - "action": "UPDATE" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : "60000", + "action" : "UPDATE" } ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-add-replicas.yaml b/src/test/resources/plans/seed-topic-add-replicas.yaml index 9a6666ba..4d702729 100644 --- a/src/test/resources/plans/seed-topic-add-replicas.yaml +++ b/src/test/resources/plans/seed-topic-add-replicas.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-2-plan.json b/src/test/resources/plans/seed-topic-modification-2-plan.json index 2064a26b..088321e6 100644 --- a/src/test/resources/plans/seed-topic-modification-2-plan.json +++ b/src/test/resources/plans/seed-topic-modification-2-plan.json @@ -1,82 +1,83 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": null, - "previousValue": "compact", - "action": "REMOVE" + "key" : "cleanup.policy", + "value" : null, + "previousValue" : "compact", + "action" : "REMOVE" }, { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-2", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification-2.yaml b/src/test/resources/plans/seed-topic-modification-2.yaml index de2f4f37..d0fd7637 100644 --- a/src/test/resources/plans/seed-topic-modification-2.yaml +++ b/src/test/resources/plans/seed-topic-modification-2.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-3-plan.json b/src/test/resources/plans/seed-topic-modification-3-plan.json index 07ad15e0..129cb086 100644 --- a/src/test/resources/plans/seed-topic-modification-3-plan.json +++ b/src/test/resources/plans/seed-topic-modification-3-plan.json @@ -1,83 +1,84 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-1", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "segment.bytes", - "value": null, - "previousValue": "100000", - "action": "REMOVE" + "key" : "segment.bytes", + "value" : null, + "previousValue" : "100000", + "action" : "REMOVE" }, { - "key": "retention.ms", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "topic-with-configs-2", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "topic-with-configs-2", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "100000", - "previousValue": "60000", - "action": "UPDATE" + "key" : "retention.ms", + "value" : "100000", + "previousValue" : "60000", + "action" : "UPDATE" } ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification-3.yaml b/src/test/resources/plans/seed-topic-modification-3.yaml index 322eca5b..dd5dda4c 100644 --- a/src/test/resources/plans/seed-topic-modification-3.yaml +++ b/src/test/resources/plans/seed-topic-modification-3.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-no-delete-plan.json b/src/test/resources/plans/seed-topic-modification-no-delete-plan.json index 66ba7b10..3bf44322 100644 --- a/src/test/resources/plans/seed-topic-modification-no-delete-plan.json +++ b/src/test/resources/plans/seed-topic-modification-no-delete-plan.json @@ -1,31 +1,32 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification-no-delete.yaml b/src/test/resources/plans/seed-topic-modification-no-delete.yaml index 1e62856f..dfddd461 100644 --- a/src/test/resources/plans/seed-topic-modification-no-delete.yaml +++ b/src/test/resources/plans/seed-topic-modification-no-delete.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-modification-plan.json b/src/test/resources/plans/seed-topic-modification-plan.json index e9175241..853e900d 100644 --- a/src/test/resources/plans/seed-topic-modification-plan.json +++ b/src/test/resources/plans/seed-topic-modification-plan.json @@ -1,63 +1,64 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "UPDATE", - "topicDetailsPlan": null, - "topicConfigPlans": [ + "name" : "test-topic", + "action" : "UPDATE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ { - "key": "retention.ms", - "value": "60000", - "previousValue": null, - "action": "ADD" + "key" : "retention.ms", + "value" : "60000", + "previousValue" : null, + "action" : "ADD" } ] }, { - "name": "new-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "new-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "delete-topic", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "delete-topic", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-2", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-2", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] }, { - "name": "topic-with-configs-1", - "action": "REMOVE", - "topicDetailsPlan": null, - "topicConfigPlans": [] + "name" : "topic-with-configs-1", + "action" : "REMOVE", + "topicDetailsPlan" : null, + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "Unnamed ACL", - "aclDetails": { - "name": "test-topic", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "Unnamed ACL", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "REMOVE" + "action" : "REMOVE" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/seed-topic-modification.yaml b/src/test/resources/plans/seed-topic-modification.yaml index 1e62856f..dfddd461 100644 --- a/src/test/resources/plans/seed-topic-modification.yaml +++ b/src/test/resources/plans/seed-topic-modification.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 1 diff --git a/src/test/resources/plans/seed-topic-remove-replicas-plan.json b/src/test/resources/plans/seed-topic-remove-replicas-plan.json index 808cd811..29bcbb32 100644 --- a/src/test/resources/plans/seed-topic-remove-replicas-plan.json +++ b/src/test/resources/plans/seed-topic-remove-replicas-plan.json @@ -27,6 +27,7 @@ "topicConfigPlans": [] } ], + "schemaPlans" : [ ], "aclPlans": [ { "name": "Unnamed ACL", diff --git a/src/test/resources/plans/seed-topic-remove-replicas.yaml b/src/test/resources/plans/seed-topic-remove-replicas.yaml index 8b472de7..f8f19a3f 100644 --- a/src/test/resources/plans/seed-topic-remove-replicas.yaml +++ b/src/test/resources/plans/seed-topic-remove-replicas.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: delete-topic: partitions: 1 diff --git a/src/test/resources/plans/simple-plan.json b/src/test/resources/plans/simple-plan.json index e841e1a1..03744be7 100644 --- a/src/test/resources/plans/simple-plan.json +++ b/src/test/resources/plans/simple-plan.json @@ -1,18 +1,19 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } \ No newline at end of file diff --git a/src/test/resources/plans/simple-users-plan.json b/src/test/resources/plans/simple-users-plan.json index f1702e15..60105020 100644 --- a/src/test/resources/plans/simple-users-plan.json +++ b/src/test/resources/plans/simple-users-plan.json @@ -1,123 +1,124 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "test-topic", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "test-topic", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-user-0", - "aclDetails": { - "name": "kafka-cluster", - "type": "CLUSTER", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "test-user-0", + "aclDetails" : { + "name" : "kafka-cluster", + "type" : "CLUSTER", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-1", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "test-user-1", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-2", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE_CONFIGS", - "permission": "ALLOW" + "name" : "test-user-2", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE_CONFIGS", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-3", - "aclDetails": { - "name": "*", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-3", + "aclDetails" : { + "name" : "*", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-4", - "aclDetails": { - "name": "*", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "DESCRIBE", - "permission": "ALLOW" + "name" : "test-user-4", + "aclDetails" : { + "name" : "*", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "DESCRIBE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-5", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-5", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-6", - "aclDetails": { - "name": "*", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-user-6", + "aclDetails" : { + "name" : "*", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-user-7", - "aclDetails": { - "name": "*", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-user-7", + "aclDetails" : { + "name" : "*", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/simple-users.yaml b/src/test/resources/plans/simple-users.yaml index 3b5eafd0..6a15658d 100644 --- a/src/test/resources/plans/simple-users.yaml +++ b/src/test/resources/plans/simple-users.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/simple.yaml b/src/test/resources/plans/simple.yaml index a4e985b0..fd59d299 100644 --- a/src/test/resources/plans/simple.yaml +++ b/src/test/resources/plans/simple.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: test-topic: partitions: 6 diff --git a/src/test/resources/plans/skip-acls-apply-plan.json b/src/test/resources/plans/skip-acls-apply-plan.json index 7a48cd63..7612c355 100644 --- a/src/test/resources/plans/skip-acls-apply-plan.json +++ b/src/test/resources/plans/skip-acls-apply-plan.json @@ -1,110 +1,111 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "MY_TOPIC", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "MY_TOPIC", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another.topic.0", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another.topic.0", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "ADD" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "ADD" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "MY_TOPIC", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "MY_TOPIC", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-1", - "aclDetails": { - "name": "my-other-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-1", + "aclDetails" : { + "name" : "my-other-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } diff --git a/src/test/resources/plans/skip-acls-plan.json b/src/test/resources/plans/skip-acls-plan.json index c70d8bc9..f04bbb09 100644 --- a/src/test/resources/plans/skip-acls-plan.json +++ b/src/test/resources/plans/skip-acls-plan.json @@ -1,44 +1,45 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "MY_TOPIC", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "MY_TOPIC", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another.topic.0", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 1, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another.topic.0", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 1, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "ADD" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "ADD" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [] + "schemaPlans" : [ ], + "aclPlans" : [ ] } diff --git a/src/test/resources/plans/skip-acls.yaml b/src/test/resources/plans/skip-acls.yaml index d52c0a41..6335ac75 100644 --- a/src/test/resources/plans/skip-acls.yaml +++ b/src/test/resources/plans/skip-acls.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: MY_TOPIC: partitions: 6 diff --git a/src/test/resources/plans/topics-and-services-plan.json b/src/test/resources/plans/topics-and-services-plan.json index ebccf613..265fb98b 100644 --- a/src/test/resources/plans/topics-and-services-plan.json +++ b/src/test/resources/plans/topics-and-services-plan.json @@ -1,110 +1,111 @@ { - "topicPlans": [ + "topicPlans" : [ { - "name": "MY_TOPIC", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 6, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "MY_TOPIC", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 6, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [] + "topicConfigPlans" : [ ] }, { - "name": "another.topic.0", - "action": "ADD", - "topicDetailsPlan": { - "partitions": 1, - "previousPartitions": null, - "partitionsAction": "ADD", - "replication": 2, - "previousReplication": null, - "replicationAction": "ADD" + "name" : "another.topic.0", + "action" : "ADD", + "topicDetailsPlan" : { + "partitions" : 1, + "previousPartitions" : null, + "partitionsAction" : "ADD", + "replication" : 2, + "previousReplication" : null, + "replicationAction" : "ADD" }, - "topicConfigPlans": [ + "topicConfigPlans" : [ { - "key": "cleanup.policy", - "value": "compact", - "previousValue": null, - "action": "ADD" + "key" : "cleanup.policy", + "value" : "compact", + "previousValue" : null, + "action" : "ADD" }, { - "key": "segment.bytes", - "value": "100000", - "previousValue": null, - "action": "ADD" + "key" : "segment.bytes", + "value" : "100000", + "previousValue" : null, + "action" : "ADD" } ] } ], - "aclPlans": [ + "schemaPlans" : [ ], + "aclPlans" : [ { - "name": "test-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "WRITE", - "permission": "ALLOW" + "name" : "test-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-1", - "aclDetails": { - "name": "MY_TOPIC", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-1", + "aclDetails" : { + "name" : "MY_TOPIC", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "test-service-2", - "aclDetails": { - "name": "test-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-0", - "aclDetails": { - "name": "another.topic.0", - "type": "TOPIC", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-0", + "aclDetails" : { + "name" : "another.topic.0", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" }, { - "name": "my-other-service-1", - "aclDetails": { - "name": "my-other-service", - "type": "GROUP", - "pattern": "LITERAL", - "principal": "User:test", - "host": "*", - "operation": "READ", - "permission": "ALLOW" + "name" : "my-other-service-1", + "aclDetails" : { + "name" : "my-other-service", + "type" : "GROUP", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" }, - "action": "ADD" + "action" : "ADD" } ] } \ No newline at end of file diff --git a/src/test/resources/plans/topics-and-services.yaml b/src/test/resources/plans/topics-and-services.yaml index db878157..2ba4950f 100644 --- a/src/test/resources/plans/topics-and-services.yaml +++ b/src/test/resources/plans/topics-and-services.yaml @@ -1,3 +1,9 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + topics: MY_TOPIC: partitions: 6 From 7749e132e3a8b771c7c3ac06524a436761aec220 Mon Sep 17 00:00:00 2001 From: Jerome Revillard Date: Tue, 19 Apr 2022 18:51:00 +0200 Subject: [PATCH 05/10] Fix Schema Registry usage with references --- Dockerfile | 2 +- build.gradle | 44 ++--- docker/config/registry_jaas.conf | 5 + docker/docker-compose.yml | 54 ++++-- .../devshawn/kafka/gitops/MainCommand.java | 2 +- .../devshawn/kafka/gitops/StateManager.java | 26 +-- .../kafka/gitops/cli/ApplyCommand.java | 2 + .../kafka/gitops/cli/PlanCommand.java | 2 + .../config/SchemaRegistryConfigLoader.java | 65 +++----- .../kafka/gitops/domain/plan/SchemaPlan.java | 4 + .../gitops/domain/state/ReferenceDetails.java | 8 +- .../gitops/domain/state/SchemaDetails.java | 20 +-- .../domain/state/settings/SettingsSchema.java | 1 - .../kafka/gitops/enums/SchemaType.java | 5 + .../kafka/gitops/manager/ApplyManager.java | 7 +- .../kafka/gitops/manager/PlanManager.java | 25 ++- .../kafka/gitops/service/ParserService.java | 2 +- .../gitops/service/SchemaRegistryService.java | 156 +++++++++++------- .../kafka/gitops/util/HelperUtil.java | 60 +++++++ .../devshawn/kafka/gitops/util/LogUtil.java | 22 ++- .../gitops/ApplyCommandIntegrationSpec.groovy | 4 +- .../gitops/PlanCommandIntegrationSpec.groovy | 6 +- .../devshawn/kafka/gitops/TestUtils.groovy | 53 +++--- .../config/KafkaGitopsConfigLoaderSpec.groovy | 4 +- ...p-id-application-prefixed-apply-output.txt | 45 +++++ ...om-group-id-application-prefixed-plan.json | 45 +++++ .../custom-group-id-application-prefixed.yaml | 14 ++ .../invalid-missing-type-output.txt | 2 +- .../invalid-modify-type-output.txt | 2 +- .../schema_registry/invalid-modify-type.yaml | 3 +- .../plans/schema_registry/no-changes.yaml | 5 +- .../schema-registry-default-apply-output.txt | 4 +- .../schema-registry-mix-apply-output.txt | 6 +- .../schema-registry-new-avro-apply-output.txt | 2 +- .../schema-registry-new-json-apply-output.txt | 2 +- ...schema-registry-new-proto-apply-output.txt | 2 +- ...schema-add-with-reference-apply-output.txt | 15 ++ .../seed-schema-add-with-reference-plan.json | 33 ++++ ...ml => seed-schema-add-with-reference.yaml} | 4 + ...eed-schema-modification-2-apply-output.txt | 2 +- .../seed-schema-modification-2.yaml | 3 +- ...eed-schema-modification-4-apply-output.txt | 2 +- .../seed-schema-modification-4.yaml | 3 +- .../seed-schema-modification-apply-output.txt | 4 +- ...ma-modification-no-delete-apply-output.txt | 4 +- src/test/resources/plans/skip-acls-plan.json | 2 +- 46 files changed, 539 insertions(+), 244 deletions(-) create mode 100644 docker/config/registry_jaas.conf create mode 100644 src/main/java/com/devshawn/kafka/gitops/enums/SchemaType.java create mode 100644 src/test/resources/plans/custom-group-id-application-prefixed-apply-output.txt create mode 100644 src/test/resources/plans/custom-group-id-application-prefixed-plan.json create mode 100644 src/test/resources/plans/custom-group-id-application-prefixed.yaml create mode 100644 src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/seed-schema-add-with-reference-plan.json rename src/test/resources/plans/schema_registry/{add-with-reference.yaml => seed-schema-add-with-reference.yaml} (66%) diff --git a/Dockerfile b/Dockerfile index c8f7f8c4..9be336b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,4 +8,4 @@ RUN apt-get update && apt-get --yes upgrade && \ apt-get install -y python3 python3-pip curl && \ rm -rf /var/lib/apt/lists/* -COPY --from=build /home/gradle/src/build/output/kafka-gitops /usr/local/bin/kafka-gitops +COPY --from=build /home/gradle/src/build/output/kafka-gitops /usr/local/bin/kafka-gitops \ No newline at end of file diff --git a/build.gradle b/build.gradle index f3d9d024..76121d53 100644 --- a/build.gradle +++ b/build.gradle @@ -22,33 +22,33 @@ repositories { maven { url "https://packages.confluent.io/maven/" } + maven { + url "https://jitpack.io" + } } dependencies { - compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.4.0' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1' - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.8" - compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.10.2" - compile 'info.picocli:picocli:4.1.4' - - implementation ('io.confluent:kafka-schema-registry-client:6.1.1') -<<<<<<< HEAD - implementation('com.flipkart.zjsonpatch:zjsonpatch:0.4.11') -======= - implementation ('io.confluent:kafka-json-schema-provider:6.1.1') - implementation ('io.confluent:kafka-protobuf-serializer:6.1.1') ->>>>>>> a208542 (Tests) - - compile 'org.slf4j:slf4j-api:1.7.30' - compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' - compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.3' + compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.8.1' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.2.2' + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.2" + compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.2" + compile 'info.picocli:picocli:4.6.3' + + implementation ('io.confluent:kafka-schema-registry-client:7.1.1') + implementation ('io.confluent:kafka-json-schema-provider:7.1.1') + implementation ('io.confluent:kafka-protobuf-serializer:7.1.1') + implementation ('io.github.java-diff-utils:java-diff-utils:4.11') + + compile 'org.slf4j:slf4j-api:1.7.36' + compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.11' + compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.11' processor 'org.inferred:freebuilder:2.7.0' - testCompile group: 'junit', name: 'junit', version: '4.12' - testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.5.14' - testCompile group: 'org.spockframework', name: 'spock-core', version: '1.2-groovy-2.5' - testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2' + testCompile group: 'junit', name: 'junit', version: '4.13.2' + testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.5.16' + testCompile group: 'org.spockframework', name: 'spock-core', version: '1.3-groovy-2.5' + testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2.2' testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0' testCompile group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.0' } @@ -88,4 +88,4 @@ task buildRelease(type: Zip, group: "build") { } buildRelease.dependsOn buildExecutableJar -buildExecutableJar.dependsOn shadowJar \ No newline at end of file +buildExecutableJar.dependsOn shadowJar diff --git a/docker/config/registry_jaas.conf b/docker/config/registry_jaas.conf new file mode 100644 index 00000000..7745b0dd --- /dev/null +++ b/docker/config/registry_jaas.conf @@ -0,0 +1,5 @@ +KafkaClient { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="test" + password="test-secret"; +}; diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 094ba014..c11161b8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,24 +1,33 @@ -version: '2.1' +version: '3.3' services: zoo1: - image: zookeeper:3.4.9 + image: zookeeper:3.8.0 hostname: zoo1 ports: - "2181:2181" + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 environment: - ZOO_MY_ID: 1 - ZOO_PORT: 2181 - ZOO_SERVERS: server.1=zoo1:2888:3888 - volumes: - - ./data/zoo1/data:/data - - ./data/zoo1/datalog:/datalog + - ZOOKEEPER_SERVER_ID=1 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_INIT_LIMIT=5 + - ZOOKEEPER_SYNC_LIMIT=2 + - ZOOKEEPER_SERVERS=zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:5.5.3 + image: confluentinc/cp-kafka:7.1.1 + user: "0:0" hostname: kafka1 ports: - "9092:9092" + restart: on-failure:3 + healthcheck: + test: ps augwwx | egrep [S]upportedKafka environment: KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT @@ -27,24 +36,28 @@ services: KAFKA_BROKER_ID: 1 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_DELETE_TOPIC_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN ZOOKEEPER_SASL_ENABLED: "false" - KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" + KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" KAFKA_SUPER_USERS: "User:test;User:kafka" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" volumes: - - ./data/kafka1/data:/var/lib/kafka/data - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf depends_on: - zoo1 kafka2: - image: confluentinc/cp-kafka:5.5.3 + image: confluentinc/cp-kafka:7.1.1 + user: "0:0" hostname: kafka2 ports: - "9093:9093" + restart: on-failure:3 + healthcheck: + test: ps augwwx | egrep [S]upportedKafka environment: KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT @@ -53,24 +66,28 @@ services: KAFKA_BROKER_ID: 2 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_DELETE_TOPIC_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN ZOOKEEPER_SASL_ENABLED: "false" - KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" + KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" KAFKA_SUPER_USERS: "User:test;User:kafka" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" volumes: - - ./data/kafka2/data:/var/lib/kafka/data - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf depends_on: - zoo1 kafka3: - image: confluentinc/cp-kafka:5.5.3 + image: confluentinc/cp-kafka:7.1.1 + user: "0:0" hostname: kafka3 ports: - "9094:9094" + restart: on-failure:3 + healthcheck: + test: ps augwwx | egrep [S]upportedKafka environment: KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT @@ -79,24 +96,25 @@ services: KAFKA_BROKER_ID: 3 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_DELETE_TOPIC_ENABLE: "true" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN ZOOKEEPER_SASL_ENABLED: "false" - KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" + KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" KAFKA_SUPER_USERS: "User:test;User:kafka" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" volumes: - - ./data/kafka3/data:/var/lib/kafka/data - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf depends_on: - zoo1 schema-registry: - image: confluentinc/cp-schema-registry:6.1.1 + image: confluentinc/cp-schema-registry:7.1.1 hostname: schema-registry ports: - "8082:8082" + restart: on-failure:5 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka1:19092,kafka2:19092,kafka3:19092" diff --git a/src/main/java/com/devshawn/kafka/gitops/MainCommand.java b/src/main/java/com/devshawn/kafka/gitops/MainCommand.java index 8aa91dc3..b653f01b 100644 --- a/src/main/java/com/devshawn/kafka/gitops/MainCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/MainCommand.java @@ -84,4 +84,4 @@ public static void main(String[] args) { int exitCode = new CommandLine(new MainCommand()).execute(args); System.exit(exitCode); } -} \ No newline at end of file +} diff --git a/src/main/java/com/devshawn/kafka/gitops/StateManager.java b/src/main/java/com/devshawn/kafka/gitops/StateManager.java index c03c6afb..e0ec618b 100644 --- a/src/main/java/com/devshawn/kafka/gitops/StateManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/StateManager.java @@ -78,7 +78,9 @@ public DesiredStateFile getAndValidateStateFile() { DesiredStateFile desiredStateFile = parserService.parseStateFile(); validateTopics(desiredStateFile); validateCustomAcls(desiredStateFile); - validateSchemas(desiredStateFile); + if (schemaRegistryService.isEnabled()) { + validateSchemas(desiredStateFile); + } this.describeAclEnabled = StateUtil.isDescribeTopicAclEnabled(desiredStateFile); return desiredStateFile; } @@ -97,7 +99,9 @@ private DesiredPlan generatePlan() { planManager.planAcls(desiredState, desiredPlan); } planManager.planTopics(desiredState, desiredPlan); - planManager.planSchemas(desiredState, desiredPlan); + if (schemaRegistryService.isEnabled()) { + planManager.planSchemas(desiredState, desiredPlan); + } return desiredPlan.build(); } @@ -113,7 +117,9 @@ public DesiredPlan apply() { if (!managerConfig.isSkipAclsDisabled()) { applyManager.applyAcls(desiredPlan); } - applyManager.applySchemas(desiredPlan); + if (schemaRegistryService.isEnabled()) { + applyManager.applySchemas(desiredPlan); + } return desiredPlan; } @@ -345,14 +351,12 @@ private void validateTopics(DesiredStateFile desiredStateFile) { private void validateSchemas(DesiredStateFile desiredStateFile) { Optional defaultSchemaCompatibility = StateUtil.fetchDefaultSchemasCompatibility(desiredStateFile); - if (!defaultSchemaCompatibility.isPresent()) { - desiredStateFile.getSchemas().forEach((subject, details) -> { - if (!details.getCompatibility().isPresent()) { - throw new ValidationException(String.format("Not set: [compatibility] in state file definition: schema -> %s", subject)); - } - schemaRegistryService.validateSchema(subject, details); - }); - } + desiredStateFile.getSchemas().forEach((subject, details) -> { + if (!defaultSchemaCompatibility.isPresent() && !details.getCompatibility().isPresent()) { + throw new ValidationException(String.format("Not set: [compatibility] in state file definition: schema -> %s", subject)); + } + //Schema parsing is deferred in the PlanManager in order not to do it more than one time. + }); } private boolean isConfluentCloudEnabled(DesiredStateFile desiredStateFile) { diff --git a/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java b/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java index 52629287..b3eeb6c1 100644 --- a/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java @@ -48,6 +48,8 @@ public Integer call() { LogUtil.printKafkaExecutionError(ex, true); } catch (SchemaRegistryExecutionException ex) { LogUtil.printSchemaRegistryExecutionError(ex, true); + } catch (Exception ex) { + LogUtil.printGenericError(ex, true); } return 2; } diff --git a/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java b/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java index caa033ca..b22e84a5 100644 --- a/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java +++ b/src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java @@ -52,6 +52,8 @@ public Integer call() { LogUtil.printPlanOutputError(ex); } catch (SchemaRegistryExecutionException ex) { LogUtil.printSchemaRegistryExecutionError(ex); + } catch (Exception ex) { + LogUtil.printGenericError(ex, false); } return 2; } diff --git a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java index d652b982..11130e65 100644 --- a/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java +++ b/src/main/java/com/devshawn/kafka/gitops/config/SchemaRegistryConfigLoader.java @@ -1,52 +1,50 @@ package com.devshawn.kafka.gitops.config; -import com.devshawn.kafka.gitops.exception.MissingConfigurationException; -import com.devshawn.kafka.gitops.exception.MissingMultipleConfigurationException; -import org.slf4j.LoggerFactory; - import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.LoggerFactory; + public class SchemaRegistryConfigLoader { private static org.slf4j.Logger log = LoggerFactory.getLogger(SchemaRegistryConfigLoader.class); public static final String SCHEMA_REGISTRY_URL_KEY = "SCHEMA_REGISTRY_URL"; public static final String SCHEMA_DIRECTORY_KEY = "SCHEMA_DIRECTORY"; - public static final String SCHEMA_REGISTRY_SASL_JAAS_USERNAME_KEY = "SCHEMA_REGISTRY_SASL_JAAS_USERNAME"; - public static final String SCHEMA_REGISTRY_SASL_JAAS_PASSWORD_KEY = "SCHEMA_REGISTRY_SASL_JAAS_PASSWORD"; - public static final String SCHEMA_REGISTRY_SASL_CONFIG_KEY = "SCHEMA_REGISTRY_SASL_CONFIG"; + public static final AtomicReference instance = new AtomicReference<>(); - private SchemaRegistryConfigLoader() {} + private SchemaRegistryConfigLoader() { + } public static SchemaRegistryConfig load() { - SchemaRegistryConfig.Builder builder = new SchemaRegistryConfig.Builder(); - setConfig(builder); - return builder.build(); + return instance.updateAndGet(v -> { + if (v != null) { + return v; + } + SchemaRegistryConfig.Builder builder = new SchemaRegistryConfig.Builder(); + setConfig(builder); + return builder.build(); + }); } private static void setConfig(SchemaRegistryConfig.Builder builder) { Map config = new HashMap<>(); - AtomicReference username = new AtomicReference<>(); - AtomicReference password = new AtomicReference<>(); Map environment = System.getenv(); environment.forEach((key, value) -> { - if (key.equals(SCHEMA_REGISTRY_SASL_JAAS_USERNAME_KEY)) { - username.set(value); - } else if (key.equals(SCHEMA_REGISTRY_SASL_JAAS_PASSWORD_KEY)) { - password.set(value); - } else if (key.equals(SCHEMA_REGISTRY_URL_KEY)) { + if (key.equals(SCHEMA_REGISTRY_URL_KEY)) { config.put(SCHEMA_REGISTRY_URL_KEY, value); } else if (key.equals(SCHEMA_DIRECTORY_KEY)) { config.put(SCHEMA_DIRECTORY_KEY, value); + } else if (key.startsWith("SCHEMA_REGISTRY_")) { + String newKey = key.substring("SCHEMA_REGISTRY_".length()).replace("_", ".").toLowerCase(); + config.put(newKey, value); } }); handleDefaultConfig(config); - handleAuthentication(username, password, config); log.info("Schema Registry Config: {}", config); @@ -54,36 +52,11 @@ private static void setConfig(SchemaRegistryConfig.Builder builder) { } private static void handleDefaultConfig(Map config) { - final String DEFAULT_URL = "http://localhost:8081"; final String CURRENT_WORKING_DIR = System.getProperty("user.dir"); - if (!config.containsKey(SCHEMA_REGISTRY_URL_KEY)) { - log.info("{} not set. Using default value of {}", SCHEMA_REGISTRY_URL_KEY, DEFAULT_URL); - config.put(SCHEMA_REGISTRY_URL_KEY, DEFAULT_URL); - } if (!config.containsKey(SCHEMA_DIRECTORY_KEY)) { - log.info("{} not set. Defaulting to current working directory: {}", SCHEMA_DIRECTORY_KEY, CURRENT_WORKING_DIR); + log.info("{} not set. Defaulting to current working directory: {}", SCHEMA_DIRECTORY_KEY, + CURRENT_WORKING_DIR); config.put(SCHEMA_DIRECTORY_KEY, CURRENT_WORKING_DIR); } } - - private static void handleAuthentication(AtomicReference username, AtomicReference password, Map config) { - if (username.get() != null && password.get() != null) { - String loginModule = "org.apache.kafka.common.security.plain.PlainLoginModule"; - String value = String.format("%s required username=\"%s\" password=\"%s\";", - loginModule, escape(username.get()), escape(password.get())); - config.put(SCHEMA_REGISTRY_SASL_CONFIG_KEY, value); - } else { - if(config.get(SCHEMA_REGISTRY_SASL_CONFIG_KEY) == null) { - log.info("{} or {} not set. No authentication configured for the Schema Registry", - SCHEMA_REGISTRY_SASL_JAAS_USERNAME_KEY, SCHEMA_REGISTRY_SASL_JAAS_PASSWORD_KEY); - } - } - } - - private static String escape(String value) { - if (value != null) { - return value.replace("\"", "\\\""); - } - return null; - } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java b/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java index 29dc53b0..58fbce3c 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/plan/SchemaPlan.java @@ -2,6 +2,7 @@ import com.devshawn.kafka.gitops.domain.state.SchemaDetails; import com.devshawn.kafka.gitops.enums.PlanAction; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.inferred.freebuilder.FreeBuilder; @@ -16,6 +17,9 @@ public interface SchemaPlan { PlanAction getAction(); Optional getSchemaDetails(); + + @JsonIgnore + Optional getDiff(); class Builder extends SchemaPlan_Builder { } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java index d8dbd41e..d201d00f 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/ReferenceDetails.java @@ -1,8 +1,10 @@ package com.devshawn.kafka.gitops.domain.state; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.inferred.freebuilder.FreeBuilder; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; @FreeBuilder @JsonDeserialize(builder = ReferenceDetails.Builder.class) @@ -14,6 +16,10 @@ public interface ReferenceDetails { Integer getVersion(); + public default SchemaReference toSchemaReference() { + return new SchemaReference(getName(), getSubject(), getVersion()); + } + class Builder extends ReferenceDetails_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java index 210d4fa1..f315ab84 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/SchemaDetails.java @@ -4,20 +4,18 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; + import org.inferred.freebuilder.FreeBuilder; + import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import com.devshawn.kafka.gitops.enums.SchemaType; import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.ValidationException; -import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import io.confluent.kafka.schemaregistry.AbstractSchemaProvider; -import io.confluent.kafka.schemaregistry.ParsedSchema; @FreeBuilder @JsonDeserialize(builder = SchemaDetails.Builder.class) @@ -33,11 +31,10 @@ public interface SchemaDetails { List getReferences(); - class Builder extends SchemaDetails_Builder { + public class Builder extends SchemaDetails_Builder { @Override public SchemaDetails build() { - AbstractSchemaProvider schemaProvider = SchemaRegistryService.schemaProviderFromType(super.getType()); - ParsedSchema parsedSchema; + String schemaRaw; if(super.getFile().isPresent()) { boolean schema = true; try { @@ -48,18 +45,17 @@ public SchemaDetails build() { if ( schema ) { throw new IllegalStateException("schema and file fields cannot be both set at the same time"); } - parsedSchema = schemaProvider.parseSchema(loadSchemaFromDisk(super.getFile().get()), Collections.emptyList()).get(); + schemaRaw = loadSchemaFromDisk(super.getFile().get()); + super.setFile(Optional.empty()); } else { - String schema; try { - schema = super.getSchema(); + schemaRaw = super.getSchema(); }catch (IllegalStateException e) { throw new IllegalStateException("schema or file field must be provided"); } - parsedSchema = schemaProvider.parseSchema(schema, Collections.emptyList()).get(); } - super.setSchema(parsedSchema.canonicalString()); + super.setSchema(schemaRaw); return super.build(); } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java index 0958915e..02fa01f2 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java @@ -1,6 +1,5 @@ package com.devshawn.kafka.gitops.domain.state.settings; -import com.devshawn.kafka.gitops.enums.SchemaCompatibility; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.inferred.freebuilder.FreeBuilder; diff --git a/src/main/java/com/devshawn/kafka/gitops/enums/SchemaType.java b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaType.java new file mode 100644 index 00000000..a045abd0 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/enums/SchemaType.java @@ -0,0 +1,5 @@ +package com.devshawn.kafka.gitops.enums; + +public enum SchemaType { + AVRO, JSON, PROTOBUF; +} diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java index 7568e8cf..3b4bafe1 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java @@ -102,11 +102,8 @@ public void applySchemas(DesiredPlan desiredPlan) { LogUtil.printSchemaPreApply(schemaPlan); schemaRegistryService.register(schemaPlan); LogUtil.printPostApply(); - } else if (schemaPlan.getAction() == PlanAction.UPDATE) { - LogUtil.printSchemaPreApply(schemaPlan); - schemaRegistryService.register(schemaPlan); - LogUtil.printPostApply(); - } else if (schemaPlan.getAction() == PlanAction.REMOVE && !managerConfig.isDeleteDisabled()) { + } else if (schemaPlan.getAction() == PlanAction.UPDATE || + (schemaPlan.getAction() == PlanAction.REMOVE && !managerConfig.isDeleteDisabled())) { LogUtil.printSchemaPreApply(schemaPlan); schemaRegistryService.deleteSubject(schemaPlan.getName(), true); LogUtil.printPostApply(); diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java index c04df2b4..708e96f1 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java @@ -24,6 +24,7 @@ import com.devshawn.kafka.gitops.domain.plan.TopicPlan; import com.devshawn.kafka.gitops.domain.state.AclDetails; import com.devshawn.kafka.gitops.domain.state.DesiredState; +import com.devshawn.kafka.gitops.domain.state.SchemaDetails; import com.devshawn.kafka.gitops.domain.state.TopicDetails; import com.devshawn.kafka.gitops.enums.PlanAction; import com.devshawn.kafka.gitops.enums.SchemaCompatibility; @@ -35,6 +36,8 @@ import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.PlanUtil; import com.fasterxml.jackson.databind.ObjectMapper; + +import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.client.SchemaMetadata; public class PlanManager { @@ -241,10 +244,15 @@ public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPl }); desiredState.getSchemas().forEach((subject, schemaDetails) -> { + // We now parse all the schemas + ParsedSchema parsedSchema = schemaRegistryService.parseSchema(subject, schemaDetails); + + //We store a properly formatted schema in the plan + SchemaDetails formattedSchemaDetails = SchemaDetails.Builder.from(schemaDetails).setSchema(parsedSchema.canonicalString()).build(); SchemaPlan.Builder schemaPlan = new SchemaPlan.Builder() .setName(subject) - .setSchemaDetails(schemaDetails); - + .setSchemaDetails(formattedSchemaDetails); + if (!currentSubjectSchemasMap.containsKey(subject)) { log.info("[PLAN] Schema Subject '{}' does not exist; it will be created.", subject); schemaPlan.setAction(PlanAction.ADD); @@ -264,14 +272,17 @@ public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPl + ", current compatibilty: " + currentCompatibility + ", new compatibilty:" + schemaDetails.getCompatibility().get() + ")"); } - boolean diff = schemaRegistryService.deepEquals(schemaDetails, currentSubjectSchemasMap.get(subject)); - if (diff) { - log.info("[PLAN] Schema Subject '{}' exists and has not changed; it will not be created.", subject); + String diff = schemaRegistryService.deepEquals(parsedSchema, currentSubjectSchemasMap.get(subject)); + if (diff.isEmpty()) { + log.info("[PLAN] Schema Subject '{}' exists and has not changed; it will not be updated.", subject); schemaPlan.setAction(PlanAction.NO_CHANGE); } else { - log.info("[PLAN] Schema Subject '{}' exists and has changed; it will be updated.", subject); + //test compatibility: + schemaRegistryService.testSchemaCompatibility(subject, parsedSchema); + log.info("[PLAN] Schema Subject '{}' exists and has changed; it will be updated. Actual diff:\n {}", subject, + diff); + schemaPlan.setDiff(diff); schemaPlan.setAction(PlanAction.UPDATE); - // TODO: Set diff string for logging? } } diff --git a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java index 9fe9a710..105692c3 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/ParserService.java @@ -64,7 +64,7 @@ public DesiredStateFile parseStateFile() { return desiredStateFile; } - public DesiredStateFile parseStateFile(File stateFile) { + private DesiredStateFile parseStateFile(File stateFile) { log.info("Parsing desired state file..."); try { diff --git a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java index 4f78c056..f10438b8 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java @@ -3,11 +3,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; -import org.apache.kafka.common.config.SaslConfigs; +import java.util.concurrent.atomic.AtomicReference; + import com.devshawn.kafka.gitops.config.SchemaRegistryConfig; import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; @@ -16,6 +15,8 @@ import com.devshawn.kafka.gitops.enums.SchemaType; import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.exception.ValidationException; +import com.devshawn.kafka.gitops.util.HelperUtil; + import io.confluent.kafka.schemaregistry.AbstractSchemaProvider; import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.SchemaProvider; @@ -25,33 +26,44 @@ import io.confluent.kafka.schemaregistry.client.rest.RestService; import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; -import io.confluent.kafka.schemaregistry.client.security.basicauth.SaslBasicAuthCredentialProvider; import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider; public class SchemaRegistryService { - - private final SchemaRegistryConfig config; + private final boolean schemaRegistryEnabled; + private static final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>(); public SchemaRegistryService(SchemaRegistryConfig config) { - this.config = config; + this.schemaRegistryEnabled = config.getConfig().containsKey(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY); + cachedSchemaRegistryClientRef.updateAndGet(v -> { + if (isEnabled()) { + if (v != null) { + return v; + } + RestService restService = new RestService( + config.getConfig().get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()); + return new CachedSchemaRegistryClient(restService, 10, config.getConfig()); + } + return null; + }); } - public Map getConfig() { - return Collections.unmodifiableMap(config.getConfig()); + public boolean isEnabled() { + return schemaRegistryEnabled; } public List getAllSubjects() { - final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); try { return new ArrayList<>(cachedSchemaRegistryClient.getAllSubjects()); } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to get all schema registry subjects", ex.getMessage()); + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get all schema registry subjects", ex.getMessage()); } } public void deleteSubject(String subject, boolean isPermanent) { - final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); try { // must always soft-delete cachedSchemaRegistryClient.deleteSubject(subject); @@ -59,24 +71,31 @@ public void deleteSubject(String subject, boolean isPermanent) { cachedSchemaRegistryClient.deleteSubject(subject, true); } } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); } } public int register(SchemaPlan schemaPlan) { - final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaPlan.getSchemaDetails().get().getType()); - ParsedSchema parsedSchema = parseSchema(schemaPlan.getName(), schemaPlan.getSchemaDetails().get(), schemaProvider, cachedSchemaRegistryClient); + ParsedSchema parsedSchema = parseSchema(schemaPlan.getName(), schemaPlan.getSchemaDetails().get(), + schemaProvider); int id; try { id = cachedSchemaRegistryClient.register(schemaPlan.getName(), parsedSchema); } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to register subject '" + schemaPlan.getName() + "' in schema registry", ex.getMessage()); + throw new SchemaRegistryExecutionException("Error thrown when attempting to register subject '" + + schemaPlan.getName() + "' in schema registry", ex.getMessage()); } try { - cachedSchemaRegistryClient.updateCompatibility(schemaPlan.getName(), schemaPlan.getSchemaDetails().get().getCompatibility().get().toString()); + cachedSchemaRegistryClient.updateCompatibility(schemaPlan.getName(), + schemaPlan.getSchemaDetails().get().getCompatibility().get().toString()); } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to update compatibility of the newly registered subject '" + schemaPlan.getName() + "' in schema registry", ex.getMessage()); + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to update compatibility of the newly registered subject '" + + schemaPlan.getName() + "' in schema registry", + ex.getMessage()); } return id; } @@ -92,105 +111,124 @@ public static AbstractSchemaProvider schemaProviderFromType(SchemaType schemaTyp } else { throw new ValidationException("Unknown schema type: " + schemaType); } + // we need to pass a schema registry client as a config because the underlying + // code can validate against the current state + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); + schemaProvider.configure( + Collections.singletonMap(SchemaProvider.SCHEMA_VERSION_FETCHER_CONFIG, schemaRegistryClient)); return schemaProvider; } - public void validateSchema(String subject, SchemaDetails schemaDetails) { + public void testSchemaCompatibility(String subject, ParsedSchema parsedSchema) { + AbstractSchemaProvider schemaProvider = schemaProviderFromType(SchemaType.valueOf(parsedSchema.schemaType())); + testSchemaCompatibility(subject, parsedSchema, schemaProvider); + } + + public ParsedSchema parseSchema(String subject, SchemaDetails schemaDetails) { AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaDetails.getType()); - validateSchema(subject, schemaDetails, schemaProvider); + return parseSchema(subject, schemaDetails, schemaProvider); } - public void validateSchema(String subject, SchemaDetails schemaDetails, AbstractSchemaProvider schemaProvider) { - CachedSchemaRegistryClient schemaRegistryClient = createSchemaRegistryClient(); - ParsedSchema parsedSchema = parseSchema(subject, schemaDetails, schemaProvider, schemaRegistryClient); + private void testSchemaCompatibility(String subject, ParsedSchema parsedSchema, AbstractSchemaProvider schemaProvider) { + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); try { List differences = schemaRegistryClient.testCompatibilityVerbose(subject, parsedSchema); - if(! differences.isEmpty()) { - throw new ValidationException(String.format("%s schema '%s' is not compatible with the latest one: %s", schemaProvider.schemaType(), subject, differences)); + if (differences != null && !differences.isEmpty()) { + throw new ValidationException(String.format("%s schema '%s' is not compatible with the latest one: %s", + schemaProvider.schemaType(), subject, differences)); } } catch (IOException | RestClientException ex) { - throw new ValidationException(String.format("Error thrown when attempting to check the compatibility of the new schema for '%s': %s", subject, ex.getMessage())); + throw new ValidationException(String.format( + "Error thrown when attempting to check the compatibility of the new schema for '%s': %s", subject, + ex.getMessage())); } } - private ParsedSchema parseSchema(String subject, SchemaDetails schemaDetails, AbstractSchemaProvider schemaProvider, CachedSchemaRegistryClient schemaRegistryClient) { + private ParsedSchema parseSchema(String subject, SchemaDetails schemaDetails, AbstractSchemaProvider schemaProvider) { Optional parsedSchema; if (schemaDetails.getReferences().isEmpty()) { parsedSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), Collections.emptyList()); if (!parsedSchema.isPresent()) { - throw new ValidationException(String.format("%s schema for subject '%s' could not be parsed.", schemaProvider.schemaType(), subject)); + throw new ValidationException(String.format("%s schema for subject '%s' could not be parsed.", + schemaProvider.schemaType(), subject)); } } else { List schemaReferences = new ArrayList<>(); schemaDetails.getReferences().forEach(referenceDetails -> { - SchemaReference schemaReference = new SchemaReference(referenceDetails.getName(), referenceDetails.getSubject(), referenceDetails.getVersion()); + SchemaReference schemaReference = new SchemaReference(referenceDetails.getName(), + referenceDetails.getSubject(), referenceDetails.getVersion()); schemaReferences.add(schemaReference); }); - // we need to pass a schema registry client as a config because the underlying code validates against the current state - schemaProvider.configure(Collections.singletonMap(SchemaProvider.SCHEMA_VERSION_FETCHER_CONFIG, schemaRegistryClient)); + try { parsedSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), schemaReferences); } catch (IllegalStateException ex) { throw new ValidationException(String.format("Reference validation error: %s", ex.getMessage())); } catch (RuntimeException ex) { - throw new ValidationException(String.format("Error thrown when attempting to validate %s schema with reference: %s", subject, ex.getMessage())); + throw new ValidationException( + String.format("Error thrown when attempting to validate %s schema with reference: %s", subject, + ex.getMessage())); } if (!parsedSchema.isPresent()) { - throw new ValidationException(String.format("%s referenced schema could not be parsed for subject %s", schemaProvider.schemaType(), subject)); + throw new ValidationException(String.format("%s referenced schema could not be parsed for subject %s", + schemaProvider.schemaType(), subject)); } } return parsedSchema.get(); } public SchemaMetadata getLatestSchemaMetadata(String subject) { - final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); try { return cachedSchemaRegistryClient.getLatestSchemaMetadata(subject); } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to get schema metadata for subject '" + subject + "'", ex.getMessage()); + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get schema metadata for subject '" + subject + "'", + ex.getMessage()); } } public SchemaCompatibility getGlobalSchemaCompatibility() { - final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); try { return SchemaCompatibility.valueOf(cachedSchemaRegistryClient.getCompatibility(null)); } catch (IOException | RestClientException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to get global schema compatibility", ex.getMessage()); + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get global schema compatibility", ex.getMessage()); } } + public SchemaCompatibility getSchemaCompatibility(String subject, SchemaCompatibility globalCompatibility) { - final CachedSchemaRegistryClient cachedSchemaRegistryClient = createSchemaRegistryClient(); + final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); try { return SchemaCompatibility.valueOf(cachedSchemaRegistryClient.getCompatibility(subject)); } catch (IOException ex) { - throw new SchemaRegistryExecutionException("Error thrown when attempting to get schema compatibility for subject '" + subject + "'", ex.getMessage()); + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get schema compatibility for subject '" + subject + "'", + ex.getMessage()); } catch (RestClientException ex) { - if(ex.getErrorCode() == 40401) { + if (ex.getErrorCode() == 40401) { return globalCompatibility; } - throw new SchemaRegistryExecutionException("Error thrown when attempting to get schema compatibility for subject '" + subject + "'", ex.getMessage()); + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get schema compatibility for subject '" + subject + "'", + ex.getMessage()); } } - public boolean deepEquals(SchemaDetails schemaDetails, SchemaMetadata schemaMetadata) { - AbstractSchemaProvider schemaProvider = schemaProviderFromType(schemaDetails.getType()); - ParsedSchema newSchema = schemaProvider.parseSchema(schemaDetails.getSchema(), Collections.emptyList()).get(); - ParsedSchema previousSchema = schemaProvider.parseSchema(schemaMetadata.getSchema(), Collections.emptyList()).get(); - return (newSchema.deepEquals(previousSchema)); - } - - public CachedSchemaRegistryClient createSchemaRegistryClient() { - RestService restService = new RestService(config.getConfig().get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()); - if(config.getConfig().get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY) != null) { - SaslBasicAuthCredentialProvider saslBasicAuthCredentialProvider = new SaslBasicAuthCredentialProvider(); - Map clientConfig = new HashMap<>(); - clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, config.getConfig() - .get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY).toString()); - saslBasicAuthCredentialProvider.configure(clientConfig); - restService.setBasicAuthCredentialProvider(saslBasicAuthCredentialProvider); + public String deepEquals(ParsedSchema newSchema, SchemaMetadata schemaMetadata) { + AbstractSchemaProvider schemaProvider = schemaProviderFromType(SchemaType.valueOf(newSchema.schemaType())); + ParsedSchema previousSchema = schemaProvider.parseSchema(schemaMetadata.getSchema(), schemaMetadata.getReferences()).get(); + + String diff = ""; + /* + * The comparison must be based on the canonical String representation otherwise + * some diff can be found even if there is no. + * The `deepEquals` function works on raw data which is not good so using the basic equals one. + */ + if (!previousSchema.equals(newSchema)) { + diff = HelperUtil.generateDiff(previousSchema, newSchema); } - return new CachedSchemaRegistryClient(restService, 10); + return diff; } - } diff --git a/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java index 103ecc15..b3ac2840 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java @@ -1,10 +1,20 @@ package com.devshawn.kafka.gitops.util; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import com.github.difflib.text.DiffRow; +import com.github.difflib.text.DiffRowGenerator; + +import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.kafka.schemaregistry.avro.AvroSchema; +import io.confluent.kafka.schemaregistry.json.JsonSchema; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; + public class HelperUtil { public static List uniqueCombine(List listOne, List listTwo) { @@ -12,4 +22,54 @@ public static List uniqueCombine(List listOne, List list set.addAll(listTwo); return new ArrayList<>(set); } + + public static String generateDiff(ParsedSchema parsedSchema1, ParsedSchema parsedSchema2) { + + List schemaStringList1 = lines(parsedSchema1); + List schemaStringList2 = lines(parsedSchema2); + + DiffRowGenerator generator = DiffRowGenerator.create() + .showInlineDiffs(false) + .inlineDiffByWord(false) + .build(); + + StringBuilder stringBuilder = new StringBuilder(); + List diffRows = generator.generateDiffRows(schemaStringList1, schemaStringList2); + for (Iterator iterator = diffRows.iterator(); iterator.hasNext();) { + if(stringBuilder.length() > 0) { + stringBuilder.append("\n"); + } + DiffRow diffRow = iterator.next(); + switch (diffRow.getTag()) { + case INSERT: + stringBuilder.append("+ ").append(diffRow.getNewLine()); + break; + case DELETE: + stringBuilder.append("- ").append(diffRow.getOldLine()); + break; + case CHANGE: + stringBuilder.append("- ").append(diffRow.getOldLine()).append("\n").append("+ ").append(diffRow.getNewLine()); + break; + default: + stringBuilder.append(diffRow.getOldLine()); + break; + } + } + + return stringBuilder.toString(); + } + + private static List lines(ParsedSchema parsedSchema) { + final String value; + if(parsedSchema instanceof AvroSchema) { + value = ((AvroSchema)parsedSchema).rawSchema().toString(true); + } else if(parsedSchema instanceof ProtobufSchema) { + value = ((ProtobufSchema)parsedSchema).rawSchema().toSchema(); + } else if(parsedSchema instanceof JsonSchema) { + value =((JsonSchema) parsedSchema).toJsonNode().toPrettyString(); + } else { + value = parsedSchema.canonicalString(); + } + return Arrays.asList(value.split("\n")); + } } diff --git a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java index 2fb78754..0847b0ed 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java @@ -64,6 +64,8 @@ private static void printTopicPlan(TopicPlan topicPlan) { System.out.println(red(String.format("- [TOPIC] %s", topicPlan.getName()))); System.out.println("\n"); break; + case NO_CHANGE: + break; } } @@ -117,7 +119,7 @@ private static void printTopicConfigPlan(TopicConfigPlan topicConfigPlan) { System.out.println(red(String.format("\t\t- %s (%s)", topicConfigPlan.getKey(), topicConfigPlan.getPreviousValue().get()))); break; case NO_CHANGE: - break; + break; } } @@ -147,6 +149,9 @@ private static void printAclPlan(AclPlan aclPlan) { System.out.println(red(String.format("\t - permission: %s", aclDetails.getPermission()))); System.out.println("\n"); break; + case UPDATE: + case NO_CHANGE: + break; } } @@ -158,11 +163,11 @@ private static void printSchemaPlan(SchemaPlan schemaPlan) { if(schemaPlan.getSchemaDetails().get().getCompatibility().isPresent()) { System.out.println(green(String.format("\t + compatibility: %s", schemaPlan.getSchemaDetails().get().getCompatibility().get()))); } - System.out.println(green(String.format("\t + schema: \n----------------------\n%s\n----------------------", + System.out.println(green(String.format("\t + schema:\n----------------------\n%s\n----------------------", schemaPlan.getSchemaDetails().get().getSchema()))); if (!schemaPlan.getSchemaDetails().get().getReferences().isEmpty()) { schemaPlan.getSchemaDetails().get().getReferences().forEach(referenceDetail -> { - System.out.println(green(String.format("\t + references:"))); + System.out.println(green("\t + references:")); System.out.println(green(String.format("\t\t + name: %s", referenceDetail.getName()))); System.out.println(green(String.format("\t\t + subject: %s", referenceDetail.getSubject()))); System.out.println(green(String.format("\t\t + version: %s", referenceDetail.getVersion()))); @@ -172,12 +177,18 @@ private static void printSchemaPlan(SchemaPlan schemaPlan) { break; case UPDATE: System.out.println(yellow(String.format("~ [SCHEMA] %s", schemaPlan.getName()))); + if(schemaPlan.getDiff().isPresent()) { + System.out.println(yellow(String.format("\t ~ diff:\n----------------------\n%s\n----------------------", + schemaPlan.getDiff().get()))); + } System.out.println("\n"); break; case REMOVE: System.out.println(red(String.format("- [SCHEMA] %s", schemaPlan.getName()))); System.out.println("\n"); break; + case NO_CHANGE: + break; } } @@ -272,7 +283,7 @@ public static void printGenericError(RuntimeException ex) { printGenericError(ex, false); } - public static void printGenericError(RuntimeException ex, boolean apply) { + public static void printGenericError(Exception ex, boolean apply) { System.out.println(String.format("[%s] %s\n", red("ERROR"), ex.getMessage())); if (apply) { printApplyErrorMessage(); @@ -357,7 +368,8 @@ private static String toAction(PlanAction planAction) { return yellow("UPDATE"); case REMOVE: return red("DELETE"); + default: + return null; } - return null; } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy index 13fa1ce7..139e2d7a 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy @@ -64,8 +64,7 @@ class ApplyCommandIntegrationSpec extends Specification { "custom-group-id-connect", "custom-application-id-streams", "custom-storage-topic", - "custom-storage-topics", - "schema-registry-simple" + "custom-storage-topics" ] } @@ -242,5 +241,6 @@ class ApplyCommandIntegrationSpec extends Specification { "seed-schema-modification-3" | false "seed-schema-modification-4" | false "seed-schema-modification" | true + "seed-schema-add-with-reference" | false } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy index 34ebc759..a3c497a4 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy @@ -105,6 +105,7 @@ class PlanCommandIntegrationSpec extends Specification { void 'test various valid plans with seed - #planName'() { setup: + TestUtils.cleanUpKafkaCluster() TestUtils.seedKafkaCluster() String planOutputFile = "/tmp/plan.json" String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") @@ -144,6 +145,7 @@ class PlanCommandIntegrationSpec extends Specification { void 'test include unchanged flag - #planNam #includeUnchanged'() { setup: + TestUtils.cleanUpKafkaCluster() TestUtils.seedKafkaCluster() String planOutputFile = "/tmp/plan.json" String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") @@ -260,6 +262,7 @@ class PlanCommandIntegrationSpec extends Specification { void 'test plan that has no changes - #includeUnchanged'() { setup: + TestUtils.cleanUpKafkaCluster() TestUtils.seedKafkaCluster() ByteArrayOutputStream out = new ByteArrayOutputStream() PrintStream oldOut = System.out @@ -311,7 +314,7 @@ class PlanCommandIntegrationSpec extends Specification { then: exitCode == 0 - + when: String actualPlan = TestUtils.getFileContent(planOutputFile) String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") @@ -362,6 +365,7 @@ class PlanCommandIntegrationSpec extends Specification { "seed-schema-modification-3" | false "seed-schema-modification-4" | false "seed-schema-modification-no-delete" | true + "seed-schema-add-with-reference" | false } void 'test schema registry plan that has no changes - #includeUnchanged'() { diff --git a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy index 892e33ed..b1daf74e 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy @@ -1,10 +1,17 @@ package com.devshawn.kafka.gitops import java.nio.file.Paths +import java.util.concurrent.atomic.AtomicReference + import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.NewTopic -import org.apache.kafka.common.acl.* +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AccessControlEntryFilter; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclBindingFilter; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.resource.PatternType import org.apache.kafka.common.resource.ResourcePattern @@ -13,22 +20,23 @@ import org.apache.kafka.common.resource.ResourceType import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader import com.devshawn.kafka.gitops.enums.SchemaCompatibility import com.devshawn.kafka.gitops.enums.SchemaType -import com.devshawn.kafka.gitops.exception.ValidationException import com.devshawn.kafka.gitops.service.SchemaRegistryService -import groovy.swing.factory.CollectionFactory import io.confluent.kafka.schemaregistry.AbstractSchemaProvider import io.confluent.kafka.schemaregistry.ParsedSchema -import io.confluent.kafka.schemaregistry.SchemaProvider -import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient import io.confluent.kafka.schemaregistry.client.rest.RestService -import io.confluent.kafka.schemaregistry.client.security.basicauth.SaslBasicAuthCredentialProvider -import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider -import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider import spock.util.concurrent.PollingConditions class TestUtils { + private static final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>(); + + static { + Map config = SchemaRegistryConfigLoader.load().getConfig(); + RestService restService = new RestService(config.get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()) + CachedSchemaRegistryClient cachedSchemaRegistryClient = new CachedSchemaRegistryClient(restService, 10, config); + cachedSchemaRegistryClientRef.set(cachedSchemaRegistryClient); + } private TestUtils() { } @@ -52,7 +60,7 @@ class TestUtils { static void cleanUpSchemaRegistry() { try { - CachedSchemaRegistryClient schemaRegistryClient = getSchemaRegistryClient(); + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); Collection subjects = schemaRegistryClient.getAllSubjects(); for (subject in subjects) { @@ -69,7 +77,7 @@ class TestUtils { static void seedSchemaRegistry() { try { - CachedSchemaRegistryClient schemaRegistryClient = getSchemaRegistryClient(); + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); createSchema("schema-1-json", SchemaType.JSON, "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}", schemaRegistryClient, SchemaCompatibility.BACKWARD) createSchema("schema-2-avro", SchemaType.AVRO, @@ -92,16 +100,17 @@ class TestUtils { } static void cleanUpKafkaCluster() { - def conditions = new PollingConditions(timeout: 120, initialDelay: 2, factor: 1.25) + def conditions = new PollingConditions(timeout: 120, initialDelay: 2, delay: 2) try { AdminClient adminClient = AdminClient.create(getKafkaConfig()) Set topics = adminClient.listTopics().names().get(); // Do not remove the schema registry topic topics.remove("_schemas"); - adminClient.deleteTopics(topics).all().get() - + adminClient.deleteTopics(topics) + conditions.eventually { + println "Testing if kafka topics still exist..." Set remainingTopics = adminClient.listTopics().names().get() assert remainingTopics.size() == 1 assert remainingTopics.getAt(0).equals("_schemas") @@ -110,9 +119,11 @@ class TestUtils { AclBindingFilter filter = getWildcardFilter() adminClient.deleteAcls(Collections.singletonList(filter)) conditions.eventually { + println "Testing if kafka acls still exist..." List acls = new ArrayList<>(adminClient.describeAcls(filter).values().get()) assert acls.size() == 0 } + adminClient.close(); println "Finished cleaning up kafka cluster" } catch (Exception ex) { println "Error cleaning up kafka cluster" @@ -139,6 +150,7 @@ class TestUtils { List newAcls = new ArrayList<>(adminClient.describeAcls(getWildcardFilter()).values().get()) assert newAcls.size() == 1 } + adminClient.close(); println "Finished seeding kafka cluster" } catch (Exception ex) { println "Error seeding up kafka cluster" @@ -153,7 +165,7 @@ class TestUtils { } static void createSchema(String subject, SchemaType type, ParsedSchema schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { - CachedSchemaRegistryClient schemaRegistryClient = getSchemaRegistryClient(); + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); int id = schemaRegistryClient.register(subject, schema); String compat = schemaRegistryClient.updateCompatibility(subject, compatibility.toString()); println "Schema subject '" + subject + "' with id " + id + " created (compatibility: " + compat + ")" @@ -195,17 +207,4 @@ class TestUtils { ] } - static CachedSchemaRegistryClient getSchemaRegistryClient() { - Map config = SchemaRegistryConfigLoader.load().getConfig(); - RestService restService = new RestService(config.get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()) - if(config.get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY) != null) { - SaslBasicAuthCredentialProvider saslBasicAuthCredentialProvider = new SaslBasicAuthCredentialProvider() - Map clientConfig = new HashMap<>() - clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, config - .get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_SASL_CONFIG_KEY).toString()) - saslBasicAuthCredentialProvider.configure(clientConfig) - restService.setBasicAuthCredentialProvider(saslBasicAuthCredentialProvider) - } - return new CachedSchemaRegistryClient(restService, 10); - } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy index a949286f..bc9e4df3 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy @@ -14,7 +14,7 @@ class KafkaGitopsConfigLoaderSpec extends Specification { EnvironmentVariables environmentVariables void setupSpec() { - environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") + environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092,localhost:9093,localhost:9094") environmentVariables.set("KAFKA_SASL_MECHANISM", "PLAIN") environmentVariables.set("KAFKA_SECURITY_PROTOCOL", "SASL_PLAINTEXT") } @@ -54,7 +54,7 @@ class KafkaGitopsConfigLoaderSpec extends Specification { then: config.config.get(CommonClientConfigs.CLIENT_ID_CONFIG) == "kafka-gitops" - config.config.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG) == "localhost:9092" + config.config.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG) == "localhost:9092,localhost:9093,localhost:9094" config.config.get(SaslConfigs.SASL_MECHANISM) == "PLAIN" } diff --git a/src/test/resources/plans/custom-group-id-application-prefixed-apply-output.txt b/src/test/resources/plans/custom-group-id-application-prefixed-apply-output.txt new file mode 100644 index 00000000..4c4b3359 --- /dev/null +++ b/src/test/resources/plans/custom-group-id-application-prefixed-apply-output.txt @@ -0,0 +1,45 @@ +Executing apply... + +Applying: [CREATE] + ++ [ACL] test-service-0 + + resource_name: test-topic + + resource_type: TOPIC + + resource_pattern: LITERAL + + resource_principal: User:test + + host: * + + operation: WRITE + + permission: ALLOW + + +Successfully applied. + +Applying: [CREATE] + ++ [ACL] test-service-1 + + resource_name: another-test-topic + + resource_type: TOPIC + + resource_pattern: LITERAL + + resource_principal: User:test + + host: * + + operation: READ + + permission: ALLOW + + +Successfully applied. + +Applying: [CREATE] + ++ [ACL] test-service-2 + + resource_name: test-service-application-prefixed + + resource_type: GROUP + + resource_pattern: PREFIXED + + resource_principal: User:test + + host: * + + operation: READ + + permission: ALLOW + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 3 created, 0 updated, 0 deleted. diff --git a/src/test/resources/plans/custom-group-id-application-prefixed-plan.json b/src/test/resources/plans/custom-group-id-application-prefixed-plan.json new file mode 100644 index 00000000..8f385130 --- /dev/null +++ b/src/test/resources/plans/custom-group-id-application-prefixed-plan.json @@ -0,0 +1,45 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ ], + "aclPlans" : [ + { + "name" : "test-service-0", + "aclDetails" : { + "name" : "test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "WRITE", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "test-service-1", + "aclDetails" : { + "name" : "another-test-topic", + "type" : "TOPIC", + "pattern" : "LITERAL", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + }, + { + "name" : "test-service-2", + "aclDetails" : { + "name" : "test-service-application-prefixed", + "type" : "GROUP", + "pattern" : "PREFIXED", + "principal" : "User:test", + "host" : "*", + "operation" : "READ", + "permission" : "ALLOW" + }, + "action" : "ADD" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/custom-group-id-application-prefixed.yaml b/src/test/resources/plans/custom-group-id-application-prefixed.yaml new file mode 100644 index 00000000..4bd84c3e --- /dev/null +++ b/src/test/resources/plans/custom-group-id-application-prefixed.yaml @@ -0,0 +1,14 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas +services: + test-service: + type: application + group-id: test-service-application-prefixed* + principal: User:test + produces: + - test-topic + consumes: + - another-test-topic \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt b/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt index 0996def9..ad6a0f38 100644 --- a/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt +++ b/src/test/resources/plans/schema_registry/invalid-missing-type-output.txt @@ -1,3 +1,3 @@ Generating execution plan... -[INVALID] type not set in state file definition: schemas -> schema-1-json +[INVALID] Not set: [type] in state file definition: schemas -> schema-1-json diff --git a/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt index 0bcf228f..0280804e 100644 --- a/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt +++ b/src/test/resources/plans/schema_registry/invalid-modify-type-output.txt @@ -1,3 +1,3 @@ Generating execution plan... -[INVALID] AVRO schema 'schema-1-json' is not compatible with the latest one: [Incompatible because of different schema type] +[INVALID] Changing the schema type is not allowed (subject: schema-1-json, current type: JSON, new type:AVRO) diff --git a/src/test/resources/plans/schema_registry/invalid-modify-type.yaml b/src/test/resources/plans/schema_registry/invalid-modify-type.yaml index 2544c8fb..d6a79471 100644 --- a/src/test/resources/plans/schema_registry/invalid-modify-type.yaml +++ b/src/test/resources/plans/schema_registry/invalid-modify-type.yaml @@ -21,4 +21,5 @@ package com.acme; message OtherRecord { int32 an_id = 1; -}' \ No newline at end of file +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/no-changes.yaml b/src/test/resources/plans/schema_registry/no-changes.yaml index 10fd50f9..d6f1571d 100644 --- a/src/test/resources/plans/schema_registry/no-changes.yaml +++ b/src/test/resources/plans/schema_registry/no-changes.yaml @@ -8,7 +8,7 @@ schemas: schema-1-json: type: JSON compatibility: BACKWARD - schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema: "{\"type\":\"object\", \"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" schema-2-avro: type: AVRO compatibility: BACKWARD @@ -21,4 +21,5 @@ package com.acme; message OtherRecord { int32 an_id = 1; -}' \ No newline at end of file +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt index cf67b708..20e85895 100644 --- a/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt +++ b/src/test/resources/plans/schema_registry/schema-registry-default-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] json-value2 + type: JSON + compatibility: FULL - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- @@ -18,7 +18,7 @@ Applying: [CREATE] + [SCHEMA] json-value3 + type: JSON + compatibility: NONE - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt index 72b3a16c..ae1d2741 100644 --- a/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt +++ b/src/test/resources/plans/schema_registry/schema-registry-mix-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] json-value + type: JSON + compatibility: FULL - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- @@ -18,7 +18,7 @@ Applying: [CREATE] + [SCHEMA] avro-value1 + type: AVRO + compatibility: NONE - + schema: + + schema: ---------------------- {"type":"record","name":"TestRecord","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello","type":"string"}]} ---------------------- @@ -31,7 +31,7 @@ Applying: [CREATE] + [SCHEMA] avro-value2 + type: AVRO + compatibility: FORWARD - + schema: + + schema: ---------------------- {"type":"record","name":"TestRecord2","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello2","type":"string"}]} ---------------------- diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt index a5840d19..5758d549 100644 --- a/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt +++ b/src/test/resources/plans/schema_registry/schema-registry-new-avro-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] avro-value + type: AVRO + compatibility: FULL - + schema: + + schema: ---------------------- {"type":"record","name":"TestRecord","namespace":"com.devshawn.kafka.gitops","fields":[{"name":"hello","type":"string"}]} ---------------------- diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt index 995325b9..1d6b009a 100644 --- a/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt +++ b/src/test/resources/plans/schema_registry/schema-registry-new-json-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] json-value + type: JSON + compatibility: BACKWARD - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- diff --git a/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt b/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt index 8038e1aa..f12da016 100644 --- a/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt +++ b/src/test/resources/plans/schema_registry/schema-registry-new-proto-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] proto-value + type: PROTOBUF + compatibility: NONE - + schema: + + schema: ---------------------- syntax = "proto3"; package com.acme; diff --git a/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt new file mode 100644 index 00000000..c55c4506 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt @@ -0,0 +1,15 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] schema-2-json + + type: JSON + + compatibility: FORWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"$ref":"otherschema"}},"additionalProperties":false} +---------------------- + + references: + + name: otherschema + + subject: json-value + + version: 1 diff --git a/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-plan.json b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-plan.json new file mode 100644 index 00000000..0e66113d --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-plan.json @@ -0,0 +1,33 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-2-json", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ + { + "name" : "otherschema", + "subject" : "schema-1-json", + "version" : 1 + } + ] + } + }, + { + "name" : "schema-2-avro", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/add-with-reference.yaml b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference.yaml similarity index 66% rename from src/test/resources/plans/schema_registry/add-with-reference.yaml rename to src/test/resources/plans/schema_registry/seed-schema-add-with-reference.yaml index bb261662..4ae0be57 100644 --- a/src/test/resources/plans/schema_registry/add-with-reference.yaml +++ b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference.yaml @@ -5,6 +5,10 @@ settings: - _schemas schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" schema-2-json: type: JSON compatibility: FORWARD diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt index f79340d0..78dfe118 100644 --- a/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] json-value + type: JSON + compatibility: FORWARD - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml index 3cfe6317..f77acb03 100644 --- a/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-2.yaml @@ -25,4 +25,5 @@ package com.acme; message OtherRecord { int32 an_id = 1; -}' \ No newline at end of file +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt index 2384b028..a2d681b8 100644 --- a/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] schema-2-json + type: JSON + compatibility: BACKWARD - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"$ref":"otherschema"}},"additionalProperties":false} ---------------------- diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml index b471023e..170b767b 100644 --- a/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-4.yaml @@ -29,4 +29,5 @@ package com.acme; message OtherRecord { int32 an_id = 1; -}' \ No newline at end of file +} +' \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt index 38a38ffb..f165628f 100644 --- a/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] json-value + type: JSON + compatibility: BACKWARD - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- @@ -18,7 +18,7 @@ Applying: [CREATE] + [SCHEMA] json-value1 + type: JSON + compatibility: NONE - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt index 91db1a03..ccd085c1 100644 --- a/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-no-delete-apply-output.txt @@ -5,7 +5,7 @@ Applying: [CREATE] + [SCHEMA] json-value + type: JSON + compatibility: BACKWARD - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- @@ -18,7 +18,7 @@ Applying: [CREATE] + [SCHEMA] json-value1 + type: JSON + compatibility: NONE - + schema: + + schema: ---------------------- {"type":"object","properties":{"f1":{"type":"string"}}} ---------------------- diff --git a/src/test/resources/plans/skip-acls-plan.json b/src/test/resources/plans/skip-acls-plan.json index f04bbb09..acc5ef62 100644 --- a/src/test/resources/plans/skip-acls-plan.json +++ b/src/test/resources/plans/skip-acls-plan.json @@ -42,4 +42,4 @@ ], "schemaPlans" : [ ], "aclPlans" : [ ] -} +} \ No newline at end of file From a61ec4fcde5bf16f80977c749a66a4275e6ca8cf Mon Sep 17 00:00:00 2001 From: Jerome Revillard Date: Fri, 22 Apr 2022 18:23:34 +0200 Subject: [PATCH 06/10] Fix bug which was deleting instead of updating a schema --- .../java/com/devshawn/kafka/gitops/manager/ApplyManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java index 3b4bafe1..dfd10c5a 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java @@ -98,12 +98,11 @@ public void applyAcls(DesiredPlan desiredPlan) { public void applySchemas(DesiredPlan desiredPlan) { desiredPlan.getSchemaPlans().forEach(schemaPlan -> { - if (schemaPlan.getAction() == PlanAction.ADD) { + if (schemaPlan.getAction() == PlanAction.ADD || schemaPlan.getAction() == PlanAction.UPDATE) { LogUtil.printSchemaPreApply(schemaPlan); schemaRegistryService.register(schemaPlan); LogUtil.printPostApply(); - } else if (schemaPlan.getAction() == PlanAction.UPDATE || - (schemaPlan.getAction() == PlanAction.REMOVE && !managerConfig.isDeleteDisabled())) { + } else if (schemaPlan.getAction() == PlanAction.REMOVE && !managerConfig.isDeleteDisabled()) { LogUtil.printSchemaPreApply(schemaPlan); schemaRegistryService.deleteSubject(schemaPlan.getName(), true); LogUtil.printPostApply(); From 7fc31df6a5f3ef9d361dc9c06f8562a7d7ab45b9 Mon Sep 17 00:00:00 2001 From: Jerome Revillard Date: Tue, 3 May 2022 13:16:40 +0200 Subject: [PATCH 07/10] Fix schema compatibility test --- .../kafka/gitops/service/KafkaService.java | 2 +- .../gitops/service/SchemaRegistryService.java | 39 +++- .../gitops/PlanCommandIntegrationSpec.groovy | 208 ++++++++++-------- .../devshawn/kafka/gitops/TestUtils.groovy | 114 +++++----- .../invalid-modify-not-compatible-output.txt | 2 +- .../invalid-modify-not-compatible2-output.txt | 1 + .../invalid-modify-not-compatible2.yaml | 11 + .../schemas/schema-registry-schema4.avsc | 51 +++++ .../schemas/schema-registry-schema5.avsc | 56 +++++ ...schema-add-with-reference-apply-output.txt | 21 +- 10 files changed, 358 insertions(+), 147 deletions(-) create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-not-compatible2-output.txt create mode 100644 src/test/resources/plans/schema_registry/invalid-modify-not-compatible2.yaml create mode 100644 src/test/resources/plans/schema_registry/schemas/schema-registry-schema4.avsc create mode 100644 src/test/resources/plans/schema_registry/schemas/schema-registry-schema5.avsc diff --git a/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java b/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java index 54603e70..fd503738 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/KafkaService.java @@ -178,7 +178,7 @@ public Map getTopicDescription(Set topics) { } private Map getTopicDescription(Set topics, AdminClient adminClient) throws InterruptedException, ExecutionException { - return adminClient.describeTopics(topics).all().get(); + return adminClient.describeTopics(topics).allTopicNames().get(); } public Map describeConfigsForTopics(List topicNames) { diff --git a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java index f10438b8..bc1635fa 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java @@ -32,17 +32,28 @@ public class SchemaRegistryService { private final boolean schemaRegistryEnabled; private static final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>(); + + //This client must be used only when the previous client does not expose the functionality. + private static final AtomicReference schemaRegistryRestService = new AtomicReference<>(); public SchemaRegistryService(SchemaRegistryConfig config) { this.schemaRegistryEnabled = config.getConfig().containsKey(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY); - cachedSchemaRegistryClientRef.updateAndGet(v -> { + schemaRegistryRestService.updateAndGet(v -> { if (isEnabled()) { if (v != null) { return v; } - RestService restService = new RestService( + return new RestService( config.getConfig().get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()); - return new CachedSchemaRegistryClient(restService, 10, config.getConfig()); + } + return null; + }); + cachedSchemaRegistryClientRef.updateAndGet(v -> { + if (isEnabled()) { + if (v != null) { + return v; + } + return new CachedSchemaRegistryClient(schemaRegistryRestService.get(), 10, config.getConfig()); } return null; }); @@ -130,11 +141,27 @@ public ParsedSchema parseSchema(String subject, SchemaDetails schemaDetails) { } private void testSchemaCompatibility(String subject, ParsedSchema parsedSchema, AbstractSchemaProvider schemaProvider) { - CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); + RestService restService = schemaRegistryRestService.get(); try { - List differences = schemaRegistryClient.testCompatibilityVerbose(subject, parsedSchema); + /* + * WARN: this does not work for TRANSITIVE compatibility types where all the versions must be tests + * we have to wait for v7.0.0 and uses: + * List differences = restService.testCompatibility(parsedSchema.canonicalString(), parsedSchema.schemaType(), + parsedSchema.references(), subject, null, true) + */ + List differences = restService.testCompatibility(parsedSchema.canonicalString(), parsedSchema.schemaType(), + parsedSchema.references(), subject, "latest", false); if (differences != null && !differences.isEmpty()) { - throw new ValidationException(String.format("%s schema '%s' is not compatible with the latest one: %s", + /* + * There is a bug on the kafka version that we have which does not always return a reason... + * So doing it now and putting a reason if we have it. + */ + List differencesDetails = restService.testCompatibility(parsedSchema.canonicalString(), parsedSchema.schemaType(), + parsedSchema.references(), subject, "latest", true); + if(differencesDetails != null && !differencesDetails.isEmpty()) { + differences = differencesDetails; + } + throw new ValidationException(String.format("%s schema '%s' is incompatible with an earlier schema: %s", schemaProvider.schemaType(), subject, differences)); } } catch (IOException | RestClientException ex) { diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy index a3c497a4..69e2b81f 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy @@ -5,6 +5,11 @@ import java.nio.file.Paths import org.junit.ClassRule import org.junit.contrib.java.lang.system.EnvironmentVariables import org.skyscreamer.jsonassert.JSONAssert + +import com.devshawn.kafka.gitops.enums.SchemaCompatibility +import com.devshawn.kafka.gitops.enums.SchemaType + +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient import picocli.CommandLine import spock.lang.Shared import spock.lang.Specification @@ -31,7 +36,7 @@ class PlanCommandIntegrationSpec extends Specification { } void cleanupSpec() { -// TestUtils.cleanUpAll() + // TestUtils.cleanUpAll() } void 'test various valid plans - #planName'() { @@ -56,24 +61,25 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "simple", - "application-service", - "kafka-connect-service", - "kafka-streams-service", - "topics-and-services", - "multi-file", - "simple-users", - "custom-service-acls", - "custom-user-acls", - "custom-group-id-application", - "custom-group-id-connect", - "custom-application-id-streams", - "custom-storage-topic", - "custom-storage-topics", - "default-replication", - "default-replication-multiple", - "describe-topic-acl-disabled", - "describe-topic-acl-enabled" + "simple", + "application-service", + "kafka-connect-service", + "kafka-streams-service", + "topics-and-services", + "multi-file", + "simple-users", + "custom-service-acls", + "custom-user-acls", + "custom-group-id-application-prefixed", + "custom-group-id-application", + "custom-group-id-connect", + "custom-application-id-streams", + "custom-storage-topic", + "custom-storage-topics", + "default-replication", + "default-replication-multiple", + "describe-topic-acl-disabled", + "describe-topic-acl-enabled" ] } @@ -99,7 +105,7 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "skip-acls" + "skip-acls" ] } @@ -202,15 +208,15 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "invalid-missing-principal", - "invalid-topic", - "unrecognized-property", - "invalid-format", - "invalid-missing-user-principal", - "invalid-storage-topics", - "invalid-default-replication-1", - "invalid-default-replication-2", - "invalid-topic-remove-partitions" + "invalid-missing-principal", + "invalid-topic", + "unrecognized-property", + "invalid-format", + "invalid-missing-user-principal", + "invalid-storage-topics", + "invalid-default-replication-1", + "invalid-default-replication-2", + "invalid-topic-remove-partitions" ] } @@ -308,64 +314,64 @@ class PlanCommandIntegrationSpec extends Specification { String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") MainCommand mainCommand = new MainCommand() CommandLine cmd = new CommandLine(mainCommand) - + when: int exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) - + then: exitCode == 0 when: String actualPlan = TestUtils.getFileContent(planOutputFile) String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") - + then: JSONAssert.assertEquals(expectedPlan, actualPlan, true) - + where: planName << [ - "schema-registry-new-json", - "schema-registry-new-avro", - "schema-registry-new-proto", - "schema-registry-default", - "schema-registry-mix" + "schema-registry-new-json", + "schema-registry-new-avro", + "schema-registry-new-proto", + "schema-registry-default", + "schema-registry-mix" ] } void 'test various valid schema registry plans with seed - #planName'() { - setup: - TestUtils.seedSchemaRegistry() - String planOutputFile = "/tmp/plan.json" - String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") - MainCommand mainCommand = new MainCommand() - CommandLine cmd = new CommandLine(mainCommand) - - when: - int exitCode - if (deleteDisabled) { - exitCode = cmd.execute("-f", file, "--no-delete", "plan", "-o", planOutputFile) - } else { - exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) - } - - then: - exitCode == 0 - - when: - String actualPlan = TestUtils.getFileContent(planOutputFile) - String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") - - then: - JSONAssert.assertEquals(expectedPlan, actualPlan, true) - - where: - planName | deleteDisabled - "seed-schema-modification" | false - "seed-schema-modification-2" | false - "seed-schema-modification-3" | false - "seed-schema-modification-4" | false - "seed-schema-modification-no-delete" | true - "seed-schema-add-with-reference" | false + setup: + TestUtils.seedSchemaRegistry() + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode + if (deleteDisabled) { + exitCode = cmd.execute("-f", file, "--no-delete", "plan", "-o", planOutputFile) + } else { + exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + } + + then: + exitCode == 0 + + when: + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-plan.json") + + then: + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + + where: + planName | deleteDisabled + "seed-schema-modification" | false + "seed-schema-modification-2" | false + "seed-schema-modification-3" | false + "seed-schema-modification-4" | false + "seed-schema-modification-no-delete" | true + "seed-schema-add-with-reference" | false } void 'test schema registry plan that has no changes - #includeUnchanged'() { @@ -433,13 +439,13 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "invalid-type", - "invalid-missing-type", - "invalid-compatibility", - "invalid-missing-compatibility", - "invalid-unrecognized-property", - "invalid-missing-file-and-schema", - "invalid-both-file-and-schema" + "invalid-type", + "invalid-missing-type", + "invalid-compatibility", + "invalid-missing-compatibility", + "invalid-unrecognized-property", + "invalid-missing-file-and-schema", + "invalid-both-file-and-schema" ] } @@ -469,12 +475,12 @@ class PlanCommandIntegrationSpec extends Specification { where: planName << [ - "invalid-modify-type", - "invalid-modify-compatibility", - "invalid-modify-not-compatible" + "invalid-modify-type", + "invalid-modify-compatibility", + "invalid-modify-not-compatible" ] } - + void 'test various invalid schema registry plans with seed (regex) - #planName'() { setup: TestUtils.seedSchemaRegistry() @@ -487,22 +493,50 @@ class PlanCommandIntegrationSpec extends Specification { String file = TestUtils.getResourceFilePath("plans/schema_registry/${planName}.yaml") MainCommand mainCommand = new MainCommand() CommandLine cmd = new CommandLine(mainCommand) - + when: int exitCode = cmd.execute("-f", file, "plan") String pattern = TestUtils.getResourceFileContent("plans/schema_registry/${planName}-output.txt") - + then: exitCode == 2 out.toString().matches(pattern) - + cleanup: System.setErr(oldErr) System.setOut(oldOut) - + where: planName << [ - "invalid-reference" + "invalid-reference" ] } + + void 'test Specific compatibility plans with manual seed'() { + setup: + CachedSchemaRegistryClient schemaRegistryClient = TestUtils.cachedSchemaRegistryClientRef.get() + TestUtils.createSchema("test-1-avro", SchemaType.AVRO, TestUtils.getResourceFileContent("plans/schema_registry/schemas/schema-registry-schema4.avsc"), + schemaRegistryClient, SchemaCompatibility.BACKWARD) + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/invalid-modify-not-compatible2.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan") + String pattern =TestUtils.getResourceFileContent("plans/schema_registry/invalid-modify-not-compatible2-output.txt") + + then: + exitCode == 2 + out.toString().matches(pattern) + + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy index b1daf74e..b4b7a112 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy @@ -1,41 +1,45 @@ package com.devshawn.kafka.gitops import java.nio.file.Paths +import java.time.Duration import java.util.concurrent.atomic.AtomicReference import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.admin.Admin import org.apache.kafka.clients.admin.NewTopic -import org.apache.kafka.common.acl.AccessControlEntry; -import org.apache.kafka.common.acl.AccessControlEntryFilter; -import org.apache.kafka.common.acl.AclBinding; -import org.apache.kafka.common.acl.AclBindingFilter; -import org.apache.kafka.common.acl.AclOperation; -import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.acl.AccessControlEntry +import org.apache.kafka.common.acl.AccessControlEntryFilter +import org.apache.kafka.common.acl.AclBinding +import org.apache.kafka.common.acl.AclBindingFilter +import org.apache.kafka.common.acl.AclOperation +import org.apache.kafka.common.acl.AclPermissionType import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.resource.PatternType import org.apache.kafka.common.resource.ResourcePattern import org.apache.kafka.common.resource.ResourcePatternFilter import org.apache.kafka.common.resource.ResourceType + import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader import com.devshawn.kafka.gitops.enums.SchemaCompatibility import com.devshawn.kafka.gitops.enums.SchemaType import com.devshawn.kafka.gitops.service.SchemaRegistryService + import io.confluent.kafka.schemaregistry.AbstractSchemaProvider import io.confluent.kafka.schemaregistry.ParsedSchema import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient import io.confluent.kafka.schemaregistry.client.rest.RestService +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException import spock.util.concurrent.PollingConditions class TestUtils { - private static final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>(); + public static final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>() static { - Map config = SchemaRegistryConfigLoader.load().getConfig(); + Map config = SchemaRegistryConfigLoader.load().getConfig() RestService restService = new RestService(config.get(SchemaRegistryConfigLoader.SCHEMA_REGISTRY_URL_KEY).toString()) - CachedSchemaRegistryClient cachedSchemaRegistryClient = new CachedSchemaRegistryClient(restService, 10, config); - cachedSchemaRegistryClientRef.set(cachedSchemaRegistryClient); + CachedSchemaRegistryClient cachedSchemaRegistryClient = new CachedSchemaRegistryClient(restService, 1, config) + cachedSchemaRegistryClientRef.set(cachedSchemaRegistryClient) } private TestUtils() { @@ -60,14 +64,25 @@ class TestUtils { static void cleanUpSchemaRegistry() { try { - CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); - - Collection subjects = schemaRegistryClient.getAllSubjects(); - for (subject in subjects) { - schemaRegistryClient.deleteSubject(subject); - schemaRegistryClient.deleteSubject(subject, true); + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get() + Collection subjects = schemaRegistryClient.getAllSubjects() + while (subjects.size() > 0) { + for (subject in subjects) { + try { + println "Deleting " +subject + schemaRegistryClient.deleteSubject(subject) + schemaRegistryClient.deleteSubject(subject, true) + } catch (RestClientException ex) { + if(ex.errorCode == 42206) { + println "Error cleaning referenced schema (" +subject + ")... will do it later" + } else { + throw ex + } + } + } + subjects = schemaRegistryClient.getAllSubjects() + Collections.shuffle(subjects) } - assert schemaRegistryClient.getAllSubjects().size() == 0 println "Finished cleaning up schema registry" } catch (Exception ex) { println "Error cleaning up schema registry" @@ -77,45 +92,44 @@ class TestUtils { static void seedSchemaRegistry() { try { - CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); - createSchema("schema-1-json", SchemaType.JSON, - "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}", schemaRegistryClient, SchemaCompatibility.BACKWARD) + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get() + createSchema("schema-1-json", SchemaType.JSON, + "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}", schemaRegistryClient, SchemaCompatibility.BACKWARD) createSchema("schema-2-avro", SchemaType.AVRO, - "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", - schemaRegistryClient, SchemaCompatibility.BACKWARD) + "{\"type\":\"record\",\"name\":\"TestRecord\",\"namespace\":\"com.devshawn.kafka.gitops\",\"fields\":[{\"name\":\"hello\",\"type\":\"string\"}]}", + schemaRegistryClient, SchemaCompatibility.BACKWARD) createSchema("schema-3-protobuf", SchemaType.PROTOBUF, - "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", - schemaRegistryClient, SchemaCompatibility.FULL) - + "syntax = \"proto3\";\npackage com.acme;\n\nmessage OtherRecord {\n int32 an_id = 1;\n}\n", + schemaRegistryClient, SchemaCompatibility.FULL) + println "Finished seeding schema registry" } catch (Exception ex) { println "Error seeding up Schema registry" throw new RuntimeException("Error seeding up schema registry", ex) } } - + static void cleanUpAll() { - cleanUpKafkaCluster(); + cleanUpKafkaCluster() cleanUpSchemaRegistry() } static void cleanUpKafkaCluster() { - def conditions = new PollingConditions(timeout: 120, initialDelay: 2, delay: 2) + def conditions = new PollingConditions(timeout: 30, initialDelay: 2, delay: 2) try { - AdminClient adminClient = AdminClient.create(getKafkaConfig()) - Set topics = adminClient.listTopics().names().get(); + Admin adminClient = Admin.create(getKafkaConfig()) + Set topics = adminClient.listTopics().names().get() // Do not remove the schema registry topic - topics.remove("_schemas"); + topics.remove("_schemas") adminClient.deleteTopics(topics) - conditions.eventually { println "Testing if kafka topics still exist..." Set remainingTopics = adminClient.listTopics().names().get() assert remainingTopics.size() == 1 assert remainingTopics.getAt(0).equals("_schemas") } - + AclBindingFilter filter = getWildcardFilter() adminClient.deleteAcls(Collections.singletonList(filter)) conditions.eventually { @@ -123,20 +137,19 @@ class TestUtils { List acls = new ArrayList<>(adminClient.describeAcls(filter).values().get()) assert acls.size() == 0 } - adminClient.close(); + adminClient.close(Duration.ofSeconds(10)); println "Finished cleaning up kafka cluster" } catch (Exception ex) { println "Error cleaning up kafka cluster" throw new RuntimeException("Error cleaning up kafka cluster", ex) } - } static void seedKafkaCluster() { def conditions = new PollingConditions(timeout: 60, initialDelay: 2, factor: 1.25) try { - AdminClient adminClient = AdminClient.create(getKafkaConfig()) + Admin adminClient = Admin.create(getKafkaConfig()) createTopic("delete-topic", 1, adminClient) createTopic("test-topic", 1, adminClient) createTopic("topic-with-configs-1", 3, adminClient, ["cleanup.policy": "compact", "segment.bytes": "100000"]) @@ -150,7 +163,7 @@ class TestUtils { List newAcls = new ArrayList<>(adminClient.describeAcls(getWildcardFilter()).values().get()) assert newAcls.size() == 1 } - adminClient.close(); + adminClient.close() println "Finished seeding kafka cluster" } catch (Exception ex) { println "Error seeding up kafka cluster" @@ -159,23 +172,23 @@ class TestUtils { } static void createSchema(String subject, SchemaType type, String schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { - AbstractSchemaProvider schemaProvider = SchemaRegistryService.schemaProviderFromType(type); - ParsedSchema parsedSchema = schemaProvider.parseSchema(schema, Collections.emptyList()).get(); + AbstractSchemaProvider schemaProvider = SchemaRegistryService.schemaProviderFromType(type) + ParsedSchema parsedSchema = schemaProvider.parseSchema(schema, Collections.emptyList()).get() createSchema(subject, type, parsedSchema, client, compatibility) } static void createSchema(String subject, SchemaType type, ParsedSchema schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { - CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get(); - int id = schemaRegistryClient.register(subject, schema); - String compat = schemaRegistryClient.updateCompatibility(subject, compatibility.toString()); + CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get() + int id = schemaRegistryClient.register(subject, schema) + String compat = schemaRegistryClient.updateCompatibility(subject, compatibility.toString()) println "Schema subject '" + subject + "' with id " + id + " created (compatibility: " + compat + ")" } - static void createTopic(String name, int partitions, AdminClient adminClient) { + static void createTopic(String name, int partitions, Admin adminClient) { createTopic(name, partitions, adminClient, null) } - static void createTopic(String name, int partitions, AdminClient adminClient, Map configs) { + static void createTopic(String name, int partitions, Admin adminClient, Map configs) { NewTopic newTopic = new NewTopic(name, partitions, (short) 2) if (configs != null) { newTopic.configs(configs) @@ -183,7 +196,7 @@ class TestUtils { adminClient.createTopics(Collections.singletonList(newTopic)).all().get() } - static void createAcl(AdminClient adminClient) { + static void createAcl(Admin adminClient) { ResourcePattern resourcePattern = new ResourcePattern(ResourceType.TOPIC, "test-topic", PatternType.LITERAL) AccessControlEntry accessControlEntry = new AccessControlEntry("User:test", "*", AclOperation.READ, AclPermissionType.ALLOW) AclBinding aclBinding = new AclBinding(resourcePattern, accessControlEntry) @@ -200,11 +213,10 @@ class TestUtils { String jaasConfig = String.format("org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";", System.getenv("KAFKA_SASL_JAAS_USERNAME"), System.getenv("KAFKA_SASL_JAAS_PASSWORD")) return [ - (CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG): System.getenv("KAFKA_BOOTSTRAP_SERVERS"), - (CommonClientConfigs.SECURITY_PROTOCOL_CONFIG): System.getenv("KAFKA_SECURITY_PROTOCOL"), - (SaslConfigs.SASL_MECHANISM) : System.getenv("KAFKA_SASL_MECHANISM"), - (SaslConfigs.SASL_JAAS_CONFIG) : jaasConfig, + (CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG): System.getenv("KAFKA_BOOTSTRAP_SERVERS"), + (CommonClientConfigs.SECURITY_PROTOCOL_CONFIG): System.getenv("KAFKA_SECURITY_PROTOCOL"), + (SaslConfigs.SASL_MECHANISM) : System.getenv("KAFKA_SASL_MECHANISM"), + (SaslConfigs.SASL_JAAS_CONFIG) : jaasConfig, ] } - } diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt index 07a01874..8e79b8e4 100644 --- a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible-output.txt @@ -1,3 +1,3 @@ Generating execution plan... -[INVALID] JSON schema 'schema-1-json' is not compatible with the latest one: [Found incompatible change: Difference{jsonPath='#/properties/f1', type=PROPERTY_REMOVED_FROM_CLOSED_CONTENT_MODEL}] +[INVALID] JSON schema 'schema-1-json' is incompatible with an earlier schema: [Found incompatible change: Difference{jsonPath='#/properties/f1', type=PROPERTY_REMOVED_FROM_CLOSED_CONTENT_MODEL}] diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2-output.txt b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2-output.txt new file mode 100644 index 00000000..7ba4b7d6 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2-output.txt @@ -0,0 +1 @@ +(?s).*\[INVALID\] AVRO schema 'test-1-avro' is incompatible with an earlier schema: \[Incompatibility\{type:TYPE_MISMATCH, location:\/fields\/7\/type, message:reader type: INT not compatible with writer type: STRING, reader:\{"type":"int","logicalType":"date"\}, writer:"string"\}\].* \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2.yaml b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2.yaml new file mode 100644 index 00000000..81add389 --- /dev/null +++ b/src/test/resources/plans/schema_registry/invalid-modify-not-compatible2.yaml @@ -0,0 +1,11 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + test-1-avro: + type: AVRO + compatibility: BACKWARD + file: "schema-registry-schema5.avsc" diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema4.avsc b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema4.avsc new file mode 100644 index 00000000..4cde42fb --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema4.avsc @@ -0,0 +1,51 @@ +{ + "namespace": "dto", + "type": "record", + "name": "CreateOpsDTO", + "fields": [ + { + "name": "externalIds", + "type": { + "type": "array", + "items": "string" + } + }, + { + "name": "attachmentId", + "type": "string" + }, + { + "name": "deliveryOfferId", + "type": "string" + }, + { + "name": "deliveryOfferCode", + "type": "string" + }, + { + "name": "deliveryOfferName", + "type": "string" + }, + { + "name": "inherited", + "type": "boolean" + }, + { + "name": "effectDate", + "type": "int", + "logicalType": "date" + }, + { + "name": "endEffectDate", + "type": "string" + }, + { + "name": "motifEndEffectDate", + "type": "string" + }, + { + "name": "updatedBy", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/schemas/schema-registry-schema5.avsc b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema5.avsc new file mode 100644 index 00000000..204be0af --- /dev/null +++ b/src/test/resources/plans/schema_registry/schemas/schema-registry-schema5.avsc @@ -0,0 +1,56 @@ +{ + "namespace": "dto", + "type": "record", + "name": "CreateOpsDTO", + "fields": [ + { + "name": "externalIds", + "type": { + "type": "array", + "items": "string" + } + }, + { + "name": "attachmentId", + "type": "string" + }, + { + "name": "deliveryOfferId", + "type": "string" + }, + { + "name": "deliveryOfferCode", + "type": "string" + }, + { + "name": "deliveryOfferName", + "type": "string" + }, + { + "name": "inherited", + "type": "boolean" + }, + { + "name": "effectDate", + "type": { + "type": "int", + "logicalType": "date" + } + }, + { + "name": "endEffectDate", + "type": { + "type": "int", + "logicalType": "date" + } + }, + { + "name": "motifEndEffectDate", + "type": "string" + }, + { + "name": "updatedBy", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt index c55c4506..f7959038 100644 --- a/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt +++ b/src/test/resources/plans/schema_registry/seed-schema-add-with-reference-apply-output.txt @@ -11,5 +11,24 @@ Applying: [CREATE] ---------------------- + references: + name: otherschema - + subject: json-value + + subject: schema-1-json + version: 1 + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-2-avro + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 2 deleted. From 57dbbd55f68a3bd5a8a5c1444b3e39306dc53d54 Mon Sep 17 00:00:00 2001 From: Jerome REVILLARD Date: Tue, 17 May 2022 16:13:27 +0200 Subject: [PATCH 08/10] Implement schemas blacklist mechanism --- .../devshawn/kafka/gitops/StateManager.java | 15 ++++++++ .../gitops/domain/state/DesiredState.java | 2 + .../domain/state/settings/Settings.java | 2 +- ...ttingsSchema.java => SettingsSchemas.java} | 8 ++-- .../settings/SettingsSchemasBlacklist.java | 16 ++++++++ .../kafka/gitops/manager/PlanManager.java | 6 +++ .../devshawn/kafka/gitops/util/StateUtil.java | 6 +-- .../gitops/PlanCommandIntegrationSpec.groovy | 1 + .../schema-registry-default.yaml | 2 +- .../schema_registry/schema-registry-mix.yaml | 2 +- ...ed-schema-modification-blacklist-plan.json | 38 +++++++++++++++++++ .../seed-schema-modification-blacklist.yaml | 19 ++++++++++ 12 files changed, 108 insertions(+), 9 deletions(-) rename src/main/java/com/devshawn/kafka/gitops/domain/state/settings/{SettingsSchema.java => SettingsSchemas.java} (64%) create mode 100644 src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasBlacklist.java create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-blacklist-plan.json create mode 100644 src/test/resources/plans/schema_registry/seed-schema-modification-blacklist.yaml diff --git a/src/main/java/com/devshawn/kafka/gitops/StateManager.java b/src/main/java/com/devshawn/kafka/gitops/StateManager.java index e0ec618b..8b51a5a2 100644 --- a/src/main/java/com/devshawn/kafka/gitops/StateManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/StateManager.java @@ -156,10 +156,15 @@ private void createServiceAccount(String name, List serviceAccou private DesiredState getDesiredState() { DesiredStateFile desiredStateFile = getAndValidateStateFile(); + + // Topics DesiredState.Builder desiredState = new DesiredState.Builder() .addAllPrefixedTopicsToIgnore(getPrefixedTopicsToIgnore(desiredStateFile)); generateTopicsState(desiredState, desiredStateFile); + + // Schemas + desiredState.addAllPrefixedSchemasToIgnore(getPrefixedSchemasToIgnore(desiredStateFile)); generateSchemasState(desiredState, desiredStateFile); if (isConfluentCloudEnabled(desiredStateFile)) { @@ -310,6 +315,16 @@ private List getPrefixedTopicsToIgnore(DesiredStateFile desiredStateFile return topics; } + private List getPrefixedSchemasToIgnore(DesiredStateFile desiredStateFile) { + List schemas = new ArrayList<>(); + try { + schemas.addAll(desiredStateFile.getSettings().get().getSchemas().get().getBlacklist().get().getPrefixed()); + } catch (NoSuchElementException ex) { + // Do nothing, no blacklist exists + } + return schemas; + } + private GetAclOptions buildGetAclOptions(String serviceName) { return new GetAclOptions.Builder().setServiceName(serviceName).setDescribeAclEnabled(describeAclEnabled).build(); } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java index feb34a92..5e9521c9 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java @@ -18,6 +18,8 @@ public interface DesiredState { List getPrefixedTopicsToIgnore(); + List getPrefixedSchemasToIgnore(); + class Builder extends DesiredState_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java index 7f1b60e0..b94657aa 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java @@ -17,7 +17,7 @@ public interface Settings { Optional getFiles(); - Optional getSchema(); + Optional getSchemas(); class Builder extends Settings_Builder { } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemas.java similarity index 64% rename from src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java rename to src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemas.java index 02fa01f2..56b3ce6a 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchema.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemas.java @@ -6,15 +6,17 @@ import java.util.Optional; @FreeBuilder -@JsonDeserialize(builder = SettingsSchema.Builder.class) -public interface SettingsSchema { +@JsonDeserialize(builder = SettingsSchemas.Builder.class) +public interface SettingsSchemas { Optional getRegistry(); Optional getDirectory(); + Optional getBlacklist(); + Optional getDefaults(); - class Builder extends SettingsSchema_Builder { + class Builder extends SettingsSchemas_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasBlacklist.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasBlacklist.java new file mode 100644 index 00000000..b06ba546 --- /dev/null +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsSchemasBlacklist.java @@ -0,0 +1,16 @@ +package com.devshawn.kafka.gitops.domain.state.settings; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.inferred.freebuilder.FreeBuilder; + +import java.util.List; + +@FreeBuilder +@JsonDeserialize(builder = SettingsSchemasBlacklist.Builder.class) +public interface SettingsSchemasBlacklist { + + List getPrefixed(); + + class Builder extends SettingsSchemasBlacklist_Builder { + } +} diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java index 708e96f1..2d6db135 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java @@ -290,6 +290,12 @@ public void planSchemas(DesiredState desiredState, DesiredPlan.Builder desiredPl }); currentSubjectSchemasMap.forEach((subject, schemaMetadata) -> { + boolean shouldIgnore = desiredState.getPrefixedSchemasToIgnore().stream().anyMatch(it -> subject.startsWith(it)); + if (shouldIgnore) { + log.info("[PLAN] Schema Subject {} due to prefix", subject); + return; + } + if (!managerConfig.isDeleteDisabled() && desiredState.getSchemas().getOrDefault(subject, null) == null) { log.info("[PLAN] Schema Subject '{}' exists and will be remove.", subject); SchemaPlan schemaPlan = new SchemaPlan.Builder() diff --git a/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java index ffc72869..140e22a7 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java @@ -22,9 +22,9 @@ public static boolean isDescribeTopicAclEnabled(DesiredStateFile desiredStateFil } public static Optional fetchDefaultSchemasCompatibility(DesiredStateFile desiredStateFile) { - if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getSchema().isPresent() - && desiredStateFile.getSettings().get().getSchema().get().getDefaults().isPresent()) { - return desiredStateFile.getSettings().get().getSchema().get().getDefaults().get().getCompatibility(); + if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getSchemas().isPresent() + && desiredStateFile.getSettings().get().getSchemas().get().getDefaults().isPresent()) { + return desiredStateFile.getSettings().get().getSchemas().get().getDefaults().get().getCompatibility(); } return Optional.empty(); } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy index 69e2b81f..769b3cec 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy @@ -370,6 +370,7 @@ class PlanCommandIntegrationSpec extends Specification { "seed-schema-modification-2" | false "seed-schema-modification-3" | false "seed-schema-modification-4" | false + "seed-schema-modification-blacklist" | false "seed-schema-modification-no-delete" | true "seed-schema-add-with-reference" | false } diff --git a/src/test/resources/plans/schema_registry/schema-registry-default.yaml b/src/test/resources/plans/schema_registry/schema-registry-default.yaml index 8bb2d667..649428bb 100644 --- a/src/test/resources/plans/schema_registry/schema-registry-default.yaml +++ b/src/test/resources/plans/schema_registry/schema-registry-default.yaml @@ -3,7 +3,7 @@ settings: blacklist: prefixed: - _schemas - schema: + schemas: defaults: compatibility: FULL diff --git a/src/test/resources/plans/schema_registry/schema-registry-mix.yaml b/src/test/resources/plans/schema_registry/schema-registry-mix.yaml index a3a2059e..ac0a4af4 100644 --- a/src/test/resources/plans/schema_registry/schema-registry-mix.yaml +++ b/src/test/resources/plans/schema_registry/schema-registry-mix.yaml @@ -3,7 +3,7 @@ settings: blacklist: prefixed: - _schemas - schema: + schemas: defaults: compatibility: FULL diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist-plan.json b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist-plan.json new file mode 100644 index 00000000..124b4e9c --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist-plan.json @@ -0,0 +1,38 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "json-value", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "BACKWARD", + "references" : [ ] + } + }, + { + "name" : "json-value1", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}}", + "file" : null, + "compatibility" : "NONE", + "references" : [ ] + } + }, + { + "name" : "schema-1-json", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} \ No newline at end of file diff --git a/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist.yaml b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist.yaml new file mode 100644 index 00000000..3f0cf281 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-modification-blacklist.yaml @@ -0,0 +1,19 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + schemas: + blacklist: + prefixed: + - schema-2 + +schemas: + json-value: + type: JSON + compatibility: BACKWARD + schema: '{"type": "object","properties": {"f1": {"type": "string"}}}' + json-value1: + type: JSON + compatibility: NONE + file: schema-registry-schema1.json \ No newline at end of file From 4707fa49d4ce9d5c87e183ceba34f369c9141995 Mon Sep 17 00:00:00 2001 From: Jerome Revillard Date: Tue, 31 May 2022 13:38:50 +0200 Subject: [PATCH 09/10] Schema can be referenced and therefor cannot be deleted Simple logic has been introduced to retry deletion in different orders --- .../kafka/gitops/manager/ApplyManager.java | 43 +++++++++++++- .../gitops/service/SchemaRegistryService.java | 24 ++++++-- .../devshawn/kafka/gitops/util/LogUtil.java | 4 ++ .../gitops/ApplyCommandIntegrationSpec.groovy | 38 ++++++++++++ .../gitops/PlanCommandIntegrationSpec.groovy | 36 +++++++++++ .../devshawn/kafka/gitops/TestUtils.groovy | 14 ++++- ...d-schema-delete-reference-apply-output.txt | 59 +++++++++++++++++++ .../seed-schema-delete-reference-plan.json | 43 ++++++++++++++ .../seed-schema-delete-reference.yaml | 19 ++++++ 9 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/plans/schema_registry/seed-schema-delete-reference-apply-output.txt create mode 100644 src/test/resources/plans/schema_registry/seed-schema-delete-reference-plan.json create mode 100644 src/test/resources/plans/schema_registry/seed-schema-delete-reference.yaml diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java index dfd10c5a..044cb2d4 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/ApplyManager.java @@ -2,19 +2,25 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; + import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.common.Node; import org.apache.kafka.common.config.ConfigResource; + import com.devshawn.kafka.gitops.config.ManagerConfig; import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; +import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; import com.devshawn.kafka.gitops.domain.plan.TopicConfigPlan; import com.devshawn.kafka.gitops.domain.plan.TopicDetailsPlan; import com.devshawn.kafka.gitops.domain.plan.TopicPlan; import com.devshawn.kafka.gitops.enums.PlanAction; +import com.devshawn.kafka.gitops.exception.SchemaRegistryExecutionException; import com.devshawn.kafka.gitops.service.KafkaService; import com.devshawn.kafka.gitops.service.SchemaRegistryService; import com.devshawn.kafka.gitops.util.LogUtil; @@ -97,6 +103,7 @@ public void applyAcls(DesiredPlan desiredPlan) { } public void applySchemas(DesiredPlan desiredPlan) { + final List referencedSubjects = new ArrayList(); desiredPlan.getSchemaPlans().forEach(schemaPlan -> { if (schemaPlan.getAction() == PlanAction.ADD || schemaPlan.getAction() == PlanAction.UPDATE) { LogUtil.printSchemaPreApply(schemaPlan); @@ -104,9 +111,41 @@ public void applySchemas(DesiredPlan desiredPlan) { LogUtil.printPostApply(); } else if (schemaPlan.getAction() == PlanAction.REMOVE && !managerConfig.isDeleteDisabled()) { LogUtil.printSchemaPreApply(schemaPlan); - schemaRegistryService.deleteSubject(schemaPlan.getName(), true); - LogUtil.printPostApply(); + if(! schemaRegistryService.deleteSubject(schemaPlan.getName(), true)) { + referencedSubjects.add(schemaPlan); + LogUtil.printDeferredApply(); + }else { + LogUtil.printPostApply(); + } } }); + if(!referencedSubjects.isEmpty()) { + cleanUpDeferredSchemas(referencedSubjects); + } + } + + private void cleanUpDeferredSchemas(List referencedSubjects) { + System.out.println("Applying deffered actions:\n"); + int maxRetry = 1; + while ( ! referencedSubjects.isEmpty() || maxRetry >= 10) { + for (Iterator iterator = referencedSubjects.iterator(); iterator.hasNext();) { + SchemaPlan referencedSubject = iterator.next(); + LogUtil.printSchemaPreApply(referencedSubject); + if(! schemaRegistryService.deleteSubject(referencedSubject.getName(), true)) { + LogUtil.printDeferredApply(); + }else { + LogUtil.printPostApply(); + iterator.remove(); + } + } + Collections.shuffle(referencedSubjects); + maxRetry++; + } + if(referencedSubjects.isEmpty()) { + System.out.println("Deferred actions successfully applied\n"); + } else { + throw new SchemaRegistryExecutionException( + "Deferred actions did not succeed...", "At least one reference cannot be removed"); + } } } diff --git a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java index bc1635fa..d924a3bb 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java @@ -7,6 +7,9 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.devshawn.kafka.gitops.config.SchemaRegistryConfig; import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader; import com.devshawn.kafka.gitops.domain.plan.SchemaPlan; @@ -30,9 +33,11 @@ import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider; public class SchemaRegistryService { + private static final Logger log = LoggerFactory.getLogger(SchemaRegistryService.class); + private final boolean schemaRegistryEnabled; - private static final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>(); - + private final AtomicReference cachedSchemaRegistryClientRef = new AtomicReference<>(); + //This client must be used only when the previous client does not expose the functionality. private static final AtomicReference schemaRegistryRestService = new AtomicReference<>(); @@ -73,7 +78,7 @@ public List getAllSubjects() { } } - public void deleteSubject(String subject, boolean isPermanent) { + public boolean deleteSubject(String subject, boolean isPermanent) { final CachedSchemaRegistryClient cachedSchemaRegistryClient = cachedSchemaRegistryClientRef.get(); try { // must always soft-delete @@ -81,10 +86,19 @@ public void deleteSubject(String subject, boolean isPermanent) { if (isPermanent) { cachedSchemaRegistryClient.deleteSubject(subject, true); } - } catch (IOException | RestClientException ex) { + } catch (RestClientException ex) { + if(ex.getErrorCode() == 42206) { + log.debug("Error cleaning referenced schema ( {} )", subject); + return false; + } else { + throw new SchemaRegistryExecutionException( + "Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); + } + } catch (IOException ex) { throw new SchemaRegistryExecutionException( "Error thrown when attempting to get delete subject from schema registry", ex.getMessage()); } + return true; } public int register(SchemaPlan schemaPlan) { @@ -111,7 +125,7 @@ public int register(SchemaPlan schemaPlan) { return id; } - public static AbstractSchemaProvider schemaProviderFromType(SchemaType schemaType) { + public AbstractSchemaProvider schemaProviderFromType(SchemaType schemaType) { AbstractSchemaProvider schemaProvider; if (schemaType == SchemaType.AVRO) { schemaProvider = new AvroSchemaProvider(); diff --git a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java index 0847b0ed..3c8d23ae 100644 --- a/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java +++ b/src/main/java/com/devshawn/kafka/gitops/util/LogUtil.java @@ -215,6 +215,10 @@ public static void printPostApply() { System.out.println("Successfully applied.\n"); } + public static void printDeferredApply() { + System.out.println("Applied deferred...\n"); + } + /* * Helpers */ diff --git a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy index 139e2d7a..1aeef022 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy @@ -2,8 +2,16 @@ package com.devshawn.kafka.gitops import java.nio.file.Path import java.nio.file.Paths + import org.junit.Rule import org.junit.contrib.java.lang.system.EnvironmentVariables +import org.skyscreamer.jsonassert.JSONAssert + +import com.devshawn.kafka.gitops.enums.SchemaCompatibility +import com.devshawn.kafka.gitops.enums.SchemaType + +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference import picocli.CommandLine import spock.lang.Specification import spock.lang.Unroll @@ -243,4 +251,34 @@ class ApplyCommandIntegrationSpec extends Specification { "seed-schema-modification" | true "seed-schema-add-with-reference" | false } + + void 'test Specific for next deferred apply with manual seed'() { + setup: + CachedSchemaRegistryClient schemaRegistryClient = TestUtils.cachedSchemaRegistryClientRef.get() + TestUtils.seedSchemaRegistry() + TestUtils.createSchema("schema-10-json", SchemaType.JSON, "{\"type\":\"object\",\"properties\":{\"test2\":{\"type\":\"string\"}}, \"additionalProperties\": false}", + schemaRegistryClient, SchemaCompatibility.BACKWARD) + List reference = new ArrayList() + reference.add(new SchemaReference("otherschema", "schema-10-json", 1)) + List subjects = schemaRegistryClient.getAllVersions("schema-10-json"); + TestUtils.createSchema("schema-11-json", SchemaType.JSON, '{"type":"object","properties":{"test2":{"\$ref":"otherschema"}}, "additionalProperties": false}', + schemaRegistryClient, SchemaCompatibility.BACKWARD, reference) + + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldOut = System.out + System.setOut(new PrintStream(out)) + String file = TestUtils.getResourceFilePath("plans/schema_registry/seed-schema-delete-reference-plan.json") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("apply", "-p", file) + + then: + out.toString() == TestUtils.getResourceFileContent("plans/schema_registry/seed-schema-delete-reference-apply-output.txt") + exitCode == 0 + + cleanup: + System.setOut(oldOut) + } } diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy index 769b3cec..2ad14bf9 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy @@ -10,6 +10,7 @@ import com.devshawn.kafka.gitops.enums.SchemaCompatibility import com.devshawn.kafka.gitops.enums.SchemaType import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference import picocli.CommandLine import spock.lang.Shared import spock.lang.Specification @@ -536,6 +537,41 @@ class PlanCommandIntegrationSpec extends Specification { exitCode == 2 out.toString().matches(pattern) + cleanup: + System.setErr(oldErr) + System.setOut(oldOut) + } + void 'test Specific for next deferred apply with manual seed'() { + setup: + CachedSchemaRegistryClient schemaRegistryClient = TestUtils.cachedSchemaRegistryClientRef.get() + TestUtils.seedSchemaRegistry() + TestUtils.createSchema("schema-10-json", SchemaType.JSON, "{\"type\":\"object\",\"properties\":{\"test2\":{\"type\":\"string\"}}, \"additionalProperties\": false}", + schemaRegistryClient, SchemaCompatibility.BACKWARD) + List reference = new ArrayList() + reference.add(new SchemaReference("otherschema", "schema-10-json", 1)) + List subjects = schemaRegistryClient.getAllVersions("schema-10-json"); + TestUtils.createSchema("schema-11-json", SchemaType.JSON, '{"type":"object","properties":{"test2":{"\$ref":"otherschema"}}, "additionalProperties": false}', + schemaRegistryClient, SchemaCompatibility.BACKWARD, reference) + ByteArrayOutputStream err = new ByteArrayOutputStream() + ByteArrayOutputStream out = new ByteArrayOutputStream() + PrintStream oldErr = System.err + PrintStream oldOut = System.out + System.setErr(new PrintStream(err)) + System.setOut(new PrintStream(out)) + String planOutputFile = "/tmp/plan.json" + String file = TestUtils.getResourceFilePath("plans/schema_registry/seed-schema-delete-reference.yaml") + MainCommand mainCommand = new MainCommand() + CommandLine cmd = new CommandLine(mainCommand) + + when: + int exitCode = cmd.execute("-f", file, "plan", "-o", planOutputFile) + String actualPlan = TestUtils.getFileContent(planOutputFile) + String expectedPlan = TestUtils.getResourceFileContent("plans/schema_registry/seed-schema-delete-reference-plan.json") + + then: + exitCode == 0 + JSONAssert.assertEquals(expectedPlan, actualPlan, true) + cleanup: System.setErr(oldErr) System.setOut(oldOut) diff --git a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy index b4b7a112..019d0a5d 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/TestUtils.groovy @@ -19,6 +19,7 @@ import org.apache.kafka.common.resource.ResourcePattern import org.apache.kafka.common.resource.ResourcePatternFilter import org.apache.kafka.common.resource.ResourceType +import com.devshawn.kafka.gitops.config.SchemaRegistryConfig import com.devshawn.kafka.gitops.config.SchemaRegistryConfigLoader import com.devshawn.kafka.gitops.enums.SchemaCompatibility import com.devshawn.kafka.gitops.enums.SchemaType @@ -29,6 +30,7 @@ import io.confluent.kafka.schemaregistry.ParsedSchema import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient import io.confluent.kafka.schemaregistry.client.rest.RestService +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException import spock.util.concurrent.PollingConditions @@ -172,12 +174,18 @@ class TestUtils { } static void createSchema(String subject, SchemaType type, String schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { - AbstractSchemaProvider schemaProvider = SchemaRegistryService.schemaProviderFromType(type) + AbstractSchemaProvider schemaProvider = new SchemaRegistryService(SchemaRegistryConfigLoader.load()).schemaProviderFromType(type) ParsedSchema parsedSchema = schemaProvider.parseSchema(schema, Collections.emptyList()).get() - createSchema(subject, type, parsedSchema, client, compatibility) + internalCreateSchema(subject, type, parsedSchema, client, compatibility) } - static void createSchema(String subject, SchemaType type, ParsedSchema schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { + static void createSchema(String subject, SchemaType type, String schema , SchemaRegistryClient client, SchemaCompatibility compatibility, List references) { + AbstractSchemaProvider schemaProvider = new SchemaRegistryService(SchemaRegistryConfigLoader.load()).schemaProviderFromType(type) + ParsedSchema parsedSchema = schemaProvider.parseSchema(schema, references).get() + internalCreateSchema(subject, type, parsedSchema, client, compatibility) + } + + private static void internalCreateSchema(String subject, SchemaType type, ParsedSchema schema , SchemaRegistryClient client, SchemaCompatibility compatibility) { CachedSchemaRegistryClient schemaRegistryClient = cachedSchemaRegistryClientRef.get() int id = schemaRegistryClient.register(subject, schema) String compat = schemaRegistryClient.updateCompatibility(subject, compatibility.toString()) diff --git a/src/test/resources/plans/schema_registry/seed-schema-delete-reference-apply-output.txt b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-apply-output.txt new file mode 100644 index 00000000..35c5b332 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-apply-output.txt @@ -0,0 +1,59 @@ +Executing apply... + +Applying: [CREATE] + ++ [SCHEMA] schema-2-json + + type: JSON + + compatibility: FORWARD + + schema: +---------------------- +{"type":"object","properties":{"f1":{"$ref":"otherschema"}},"additionalProperties":false} +---------------------- + + references: + + name: otherschema + + subject: schema-1-json + + version: 1 + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-2-avro + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-10-json + + +Applied deferred... + +Applying: [DELETE] + +- [SCHEMA] schema-3-protobuf + + +Successfully applied. + +Applying: [DELETE] + +- [SCHEMA] schema-11-json + + +Successfully applied. + +Applying deffered actions: + +Applying: [DELETE] + +- [SCHEMA] schema-10-json + + +Successfully applied. + +Deferred actions successfully applied + +[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 4 deleted. diff --git a/src/test/resources/plans/schema_registry/seed-schema-delete-reference-plan.json b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-plan.json new file mode 100644 index 00000000..e018d715 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-delete-reference-plan.json @@ -0,0 +1,43 @@ +{ + "topicPlans" : [ ], + "schemaPlans" : [ + { + "name" : "schema-2-json", + "action" : "ADD", + "schemaDetails" : { + "type" : "JSON", + "schema" : "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}},\"additionalProperties\":false}", + "file" : null, + "compatibility" : "FORWARD", + "references" : [ + { + "name" : "otherschema", + "subject" : "schema-1-json", + "version" : 1 + } + ] + } + }, + { + "name" : "schema-2-avro", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-10-json", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-3-protobuf", + "action" : "REMOVE", + "schemaDetails" : null + }, + { + "name" : "schema-11-json", + "action" : "REMOVE", + "schemaDetails" : null + } + ], + "aclPlans" : [ ] +} diff --git a/src/test/resources/plans/schema_registry/seed-schema-delete-reference.yaml b/src/test/resources/plans/schema_registry/seed-schema-delete-reference.yaml new file mode 100644 index 00000000..4ae0be57 --- /dev/null +++ b/src/test/resources/plans/schema_registry/seed-schema-delete-reference.yaml @@ -0,0 +1,19 @@ +settings: + topics: + blacklist: + prefixed: + - _schemas + +schemas: + schema-1-json: + type: JSON + compatibility: BACKWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"type\":\"string\"}}, \"additionalProperties\": false}" + schema-2-json: + type: JSON + compatibility: FORWARD + schema: "{\"type\":\"object\",\"properties\":{\"f1\":{\"$ref\":\"otherschema\"}}, \"additionalProperties\": false}" + references: + - name: otherschema + subject: schema-1-json + version: 1 \ No newline at end of file From 0046847dd9aacd8bd9e0c286178b4ef82790104a Mon Sep 17 00:00:00 2001 From: Zbigniew BARANOWSKI Date: Fri, 28 Apr 2023 18:01:19 +0200 Subject: [PATCH 10/10] Support schema registry 7.3.x --- build.gradle | 6 +++--- docker/docker-compose.yml | 8 ++++---- .../kafka/gitops/service/SchemaRegistryService.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index 76121d53..62dbd2de 100644 --- a/build.gradle +++ b/build.gradle @@ -34,9 +34,9 @@ dependencies { compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.2" compile 'info.picocli:picocli:4.6.3' - implementation ('io.confluent:kafka-schema-registry-client:7.1.1') - implementation ('io.confluent:kafka-json-schema-provider:7.1.1') - implementation ('io.confluent:kafka-protobuf-serializer:7.1.1') + implementation ('io.confluent:kafka-schema-registry-client:7.3.3') + implementation ('io.confluent:kafka-json-schema-provider:7.3.3') + implementation ('io.confluent:kafka-protobuf-serializer:7.3.3') implementation ('io.github.java-diff-utils:java-diff-utils:4.11') compile 'org.slf4j:slf4j-api:1.7.36' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c11161b8..e7fbd59b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,7 +20,7 @@ services: - ZOOKEEPER_SERVERS=zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.3 user: "0:0" hostname: kafka1 ports: @@ -50,7 +50,7 @@ services: - zoo1 kafka2: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.3 user: "0:0" hostname: kafka2 ports: @@ -80,7 +80,7 @@ services: - zoo1 kafka3: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.3 user: "0:0" hostname: kafka3 ports: @@ -110,7 +110,7 @@ services: - zoo1 schema-registry: - image: confluentinc/cp-schema-registry:7.1.1 + image: confluentinc/cp-schema-registry:7.3.3 hostname: schema-registry ports: - "8082:8082" diff --git a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java index d924a3bb..6224fd02 100644 --- a/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java +++ b/src/main/java/com/devshawn/kafka/gitops/service/SchemaRegistryService.java @@ -248,7 +248,7 @@ public SchemaCompatibility getSchemaCompatibility(String subject, SchemaCompatib "Error thrown when attempting to get schema compatibility for subject '" + subject + "'", ex.getMessage()); } catch (RestClientException ex) { - if (ex.getErrorCode() == 40401) { + if (ex.getErrorCode() == 40401 || ex.getErrorCode() == 40408) { return globalCompatibility; } throw new SchemaRegistryExecutionException(