Update VitessTestDb to support loading the schema directory

from a filesystem path

GitOrigin-RevId: 1e6870f5c8ac859624ebe3fc9166926458c1364c

Author: aparajon

misk-vitess/README.md

@@ -25,11 +25,12 @@ This will start a minified Vitess cluster in a Docker container that will be rea
 
 ### Configuration
 
-`VitessTestDb`  depends on the existence of a schema directory ( i.e., `schemaDir`),
-which contains `.sql` schema change and `vschema.json` files.  The expected input format is shown below:
+`VitessTestDb`  depends on the existence of a schema directory ( i.e., `schemaDir`), which contains `.sql` schema change and `vschema.json` files. The location can be a classpath or filesystem path, which is designated by the prefix `classpath:` or `filesystem:`. When using `classpath:`, `VitessTestDb` looks within `resources`.
+
+The expected input format is shown below:
 
 ```
-/vitess/schema
+/resources/vitess/schema
 ├── keyspace1
 │   ├── v0001__add_table.sql
 │   ├── vschema.json
@@ -38,10 +39,16 @@ which contains `.sql` schema change and `vschema.json` files.  The expected inpu
 │   ├── vschema.json
 ```
 
-The `schemaDir` option defaults to `vitess/schema`, but can take in any custom resource path to the constructor, e.g.,
+`VitessTestDb` will throw exceptions if an invalid directory structure is provided. The default value is `classpath:/vitess/schema`, it but can take in any custom resource path to the constructor, e.g.,
+
+```
+testDb = VitessTestDb(schemaDir = "classpath:/prefix/my_schema")
+```
+
+To use a filesystem path, the prefix `filesystem:` should be used, e.g.,
 
 ```
-testDb = VitessTestDb(schemaDir = "prefix/my_schema")
+testDb = VitessTestDb(schemaDir = "filesystem:/some_path/some_path2/my_schema")
 ```
 
 Other configurable parameters include:

misk-vitess/src/test/kotlin/misk/vitess/testing/CustomArgsTest.kt

@@ -1,10 +1,5 @@
 package misk.vitess.testing
 
-import com.github.dockerjava.api.DockerClient
-import com.github.dockerjava.core.DefaultDockerClientConfig
-import com.github.dockerjava.core.DockerClientBuilder
-import com.github.dockerjava.httpclient5.ApacheDockerHttpClient
-import misk.docker.withMiskDefaults
 import misk.vitess.testing.internal.VitessClusterConfig
 import misk.vitess.testing.internal.VitessQueryExecutor
 import misk.vitess.testing.internal.VitessQueryExecutorException
@@ -43,7 +38,6 @@ class CustomArgsTest {
     private const val DB2_PORT = 34003
 
     private val executorService = Executors.newFixedThreadPool(2)
-    private val dockerClient = setupDockerClient()
 
     @JvmStatic
     @BeforeAll
@@ -56,6 +50,7 @@ class CustomArgsTest {
         port = DB1_PORT,
         keepAlive = true,
         mysqlVersion = DB1_MYSQL_VERSION,
+        schemaDir = "filesystem:${Paths.get(System.getProperty("user.dir"), "src/test/resources/vitess/schema")}",
         sqlMode = DB1_SQL_MODE,
         transactionIsolationLevel = DB1_TXN_ISO_LEVEL,
         transactionTimeoutSeconds = Duration.ofSeconds(5),
@@ -77,16 +72,6 @@ class CustomArgsTest {
       testDb1QueryExecutor = VitessQueryExecutor(VitessClusterConfig(DB1_PORT))
       testDb2QueryExecutor = VitessQueryExecutor(VitessClusterConfig(DB2_PORT))
     }
-
-    private fun setupDockerClient(): DockerClient {
-      val dockerClientConfig =
-        DefaultDockerClientConfig.createDefaultConfigBuilder().withMiskDefaults().build()
-      return DockerClientBuilder.getInstance(dockerClientConfig)
-        .withDockerHttpClient(
-          ApacheDockerHttpClient.Builder().dockerHost(dockerClientConfig.dockerHost).build()
-        )
-        .build()
-    }
   }
 
   @Test
@@ -192,6 +177,15 @@ class CustomArgsTest {
     assertTablesApplied()
   }
 
+  @Test
+  fun `test unsupported schema directory prefix `() {
+    val exception = assertThrows<VitessTestDbStartupException> { createUnsupportedSchemaDirectoryDb().applySchema() }
+    assertEquals(
+      "Schema directory `some/path/without/filesystem/or/classpath` must start with one of the supported prefixes: [classpath:, filesystem:]",
+      exception.message,
+    )
+  }
+
   private fun assertTraditionalSchemaUpdatesApplied(applySchemaResult: ApplySchemaResult) {
     // The vschema is always applied for each keyspace.
     assertEquals(2, applySchemaResult.vschemaUpdates.size)
@@ -226,4 +220,10 @@ class CustomArgsTest {
       port = DB1_PORT,
       vitessImage = "vitess/vttestserver:v19.0.9-mysql80")
   }
+
+  private fun createUnsupportedSchemaDirectoryDb(): VitessTestDb {
+    return VitessTestDb(
+      containerName = "unsupported_schema_dir_vitess_db",
+      schemaDir = "some/path/without/filesystem/or/classpath")
+  }
 }

misk-vitess/src/testFixtures/kotlin/misk/vitess/testing/VitessTestDb.kt

@@ -19,9 +19,12 @@ import kotlin.time.measureTime
  * @property keepAlive Whether to keep the database running after the test suite completes. Default is `true`.
  * @property mysqlVersion The MySQL version to use. Default is `8.0.36`.
  * @property port The port to connect to the database, which represents the vtgate. Default is `27003`.
- * @property schemaDir Location of vschema and SQL migration files in resources. The expected input format looks like:
+ * @property schemaDir The location of vschema and SQL schema change files, which can be a classpath or filesystem path, which
+ * is designated by the prefix `classpath:` or `filesystem:`. When using `classpath:`, `VitessTestDb` looks within `resources`.
+ *
+ * The expected input format of a schema directory looks like:
  * ```
- * /resources
+ * /resources/vitess
  * ├── schema
  * │   ├── keyspace1
  * │   │   ├── v0001__add_table.sql
@@ -31,7 +34,7 @@ import kotlin.time.measureTime
  * │   │   ├── vschema.json
  * ```
  *
- * In this example, schema would be the expected arg. The default is `vitess/schema`.
+ * `VitessTestDb` will throw exceptions if an invalid directory structure is provided. The default value is `classpath:/vitess/schema`.
  *
  * @property sqlMode The server SQL mode. Defaults to the MySQL8 defaults:
  *   `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION`.

misk-vitess/src/testFixtures/kotlin/misk/vitess/testing/VitessTestDbSettings.kt

@@ -12,7 +12,7 @@ object DefaultSettings {
   const val LINT_SCHEMA = false
   const val MYSQL_VERSION = "8.0.36"
   const val PORT = 27003
-  const val SCHEMA_DIR = "vitess/schema"
+  const val SCHEMA_DIR = "classpath:/vitess/schema"
   const val SQL_MODE: String =
     "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION"
   @JvmField var TRANSACTION_ISOLATION_LEVEL: TransactionIsolationLevel = TransactionIsolationLevel.REPEATABLE_READ

misk-vitess/src/testFixtures/kotlin/misk/vitess/testing/internal/VitessSchemaManager.kt

@@ -24,6 +24,7 @@ import misk.vitess.testing.DefaultSettings.VTCTLD_CLIENT_IMAGE
 import misk.vitess.testing.VSchemaUpdate
 import misk.vitess.testing.VitessTestDbStartupException
 import wisp.resources.ClasspathResourceLoaderBackend
+import wisp.resources.FilesystemLoaderBackend
 import wisp.resources.ResourceLoader
 
 /**
@@ -44,8 +45,14 @@ internal class VitessSchemaManager(
   val keyspaces: List<VitessKeyspace>
 
   init {
-    // We may be able to remove the temp schema directory logic if we can parse the schema files directly from
-    // resources.
+    val supportedSchemaPrefixes = listOf(ClasspathResourceLoaderBackend.SCHEME, FilesystemLoaderBackend.SCHEME)
+    val usesSupportedPrefix = supportedSchemaPrefixes.any { schemaDir.startsWith(it) }
+    if (!usesSupportedPrefix) {
+      throw VitessTestDbStartupException(
+        "Schema directory `$schemaDir` must start with one of the supported prefixes: $supportedSchemaPrefixes"
+      )
+    }
+
     val tempSchemaDir = createTempSchemaDirectory()
 
     currentSchemaDirPath = tempSchemaDir
@@ -326,16 +333,20 @@ internal class VitessSchemaManager(
   private fun createTempSchemaDirectory(): Path {
     val tempDir = Files.createTempDirectory("schema-")
 
-    val resourceLoader = ResourceLoader(mapOf(ClasspathResourceLoaderBackend.SCHEME to ClasspathResourceLoaderBackend))
+    val resourceLoader = ResourceLoader(
+      mapOf(
+        ClasspathResourceLoaderBackend.SCHEME to ClasspathResourceLoaderBackend,
+        FilesystemLoaderBackend.SCHEME to FilesystemLoaderBackend
+      )
+    )
 
     tempDir.createDirectories()
 
-    val resourcePath = "classpath:/$schemaDir"
-    if (!resourceLoader.exists(resourcePath)) {
-      throw VitessTestDbStartupException("Schema directory `$schemaDir` does not exist in resources")
+    if (!resourceLoader.exists(schemaDir)) {
+      throw VitessTestDbStartupException("Schema directory `$schemaDir` does not exist")
     }
 
-    resourceLoader.copyTo(resourcePath, tempDir)
+    resourceLoader.copyTo(schemaDir, tempDir)
 
     return tempDir
   }