chore(rust): add plaintext to encrypted table migration examples (#1977)

Author: rishav-karanjit

.github/workflows/library_rust_tests.yml

@@ -146,3 +146,4 @@ jobs:
         shell: bash
         run: |
           cargo run --release --example main
+          cargo test --release --example main

DynamoDbEncryption/runtimes/rust/examples/main.rs

@@ -14,6 +14,7 @@ pub mod keyring;
 pub mod multi_get_put_example;
 pub mod searchableencryption;
 pub mod test_utils;
+pub mod migration;
 
 use std::convert::From;
 

DynamoDbEncryption/runtimes/rust/examples/migration/mod.rs

@@ -0,0 +1,4 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod plaintext_to_awsdbe;

DynamoDbEncryption/runtimes/rust/examples/migration/plaintext_to_awsdbe/README.md

@@ -0,0 +1,51 @@
+# Plaintext DynamoDB Table to AWS Database Encryption SDK Encrypted Table Migration
+
+This projects demonstrates the steps necessary
+to migrate to the AWS Database Encryption SDK for DynamoDb
+from a plaintext database.
+
+[Step 0](plaintext/step0.go) demonstrates the starting state for your system.
+
+## Step 1
+
+In Step 1, you update your system to do the following:
+
+- continue to read plaintext items
+- continue to write plaintext items
+- prepare to read encrypted items
+
+When you deploy changes in Step 1,
+you should not expect any behavior change in your system,
+and your dataset still consists of plaintext data.
+
+You must ensure that the changes in Step 1 make it to all your readers before you proceed to Step 2.
+
+## Step 2
+
+In Step 2, you update your system to do the following:
+
+- continue to read plaintext items
+- start writing encrypted items
+- continue to read encrypted items
+
+When you deploy changes in Step 2,
+you are introducing encrypted items to your system,
+and must make sure that all your readers are updated with the changes from Step 1.
+
+Before you move onto the next step, you will need to encrypt all plaintext items in your dataset.
+Once you have completed this step,
+while new items are being encrypted using the new format and will be authenticated on read,
+your system will still accept reading plaintext, unauthenticated items.
+In order to complete migration to a system where you always authenticate your items,
+you should prioritize moving on to Step 3.
+
+## Step 3
+
+Once all old items are encrypted,
+update your system to do the following:
+
+- continue to write encrypted items
+- continue to read encrypted items
+- do not accept reading plaintext items
+
+Once you have deployed these changes to your system, you have completed migration.

DynamoDbEncryption/runtimes/rust/examples/migration/plaintext_to_awsdbe/awsdbe/common.rs

@@ -0,0 +1,90 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use aws_db_esdk::material_providers::client;
+use aws_db_esdk::material_providers::types::material_providers_config::MaterialProvidersConfig;
+use aws_db_esdk::CryptoAction;
+use aws_db_esdk::dynamodb::types::DynamoDbTableEncryptionConfig;
+use aws_db_esdk::types::dynamo_db_tables_encryption_config::DynamoDbTablesEncryptionConfig;
+use aws_db_esdk::dynamodb::types::PlaintextOverride;
+use std::collections::HashMap;
+
+pub async fn create_table_configs(
+    kms_key_id: &str,
+    ddb_table_name: &str,
+    plaintext_override: PlaintextOverride,
+) -> Result<DynamoDbTablesEncryptionConfig, Box<dyn std::error::Error>> {
+    // Create a Keyring. This Keyring will be responsible for protecting the data keys that protect your data.
+    // For this example, we will create a AWS KMS Keyring with the AWS KMS Key we want to use.
+    // We will use the `CreateMrkMultiKeyring` method to create this keyring,
+    // as it will correctly handle both single region and Multi-Region KMS Keys.
+    let provider_config = MaterialProvidersConfig::builder().build()?;
+    let mat_prov = client::Client::from_conf(provider_config)?;
+    let kms_keyring = mat_prov
+        .create_aws_kms_mrk_multi_keyring()
+        .generator(kms_key_id)
+        .send()
+        .await?;
+
+    // Configure which attributes are encrypted and/or signed when writing new items.
+    // For each attribute that may exist on the items we plan to write to our DynamoDbTable,
+    // we must explicitly configure how they should be treated during item encryption:
+    //  - ENCRYPT_AND_SIGN: The attribute is encrypted and included in the signature
+    //  - SIGN_ONLY: The attribute not encrypted, but is still included in the signature
+    //  - DO_NOTHING: The attribute is not encrypted and not included in the signature
+    let partition_key_name = "partition_key";
+    let sort_key_name = "sort_key";
+    let attribute_actions_on_encrypt = HashMap::from([
+        (partition_key_name.to_string(), CryptoAction::SignOnly),
+        (sort_key_name.to_string(), CryptoAction::SignOnly),
+        ("attribute1".to_string(), CryptoAction::EncryptAndSign),
+        ("attribute2".to_string(), CryptoAction::SignOnly),
+        ("attribute3".to_string(), CryptoAction::DoNothing),
+    ]);
+
+    // Configure which attributes we expect to be excluded in the signature
+    // when reading items. There are two options for configuring this:
+    //
+    //    - (Recommended) Configure `allowedUnsignedAttributesPrefix`:
+    //      When defining your DynamoDb schema and deciding on attribute names,
+    //      choose a distinguishing prefix (such as ":") for all attributes that
+    //      you do not want to include in the signature.
+    //      This has two main benefits:
+    //      - It is easier to reason about the security and authenticity of data within your item
+    //        when all unauthenticated data is easily distinguishable by their attribute name.
+    //      - If you need to add new unauthenticated attributes in the future,
+    //        you can easily make the corresponding update to your `attributeActionsOnEncrypt`
+    //        and immediately start writing to that new attribute, without
+    //        any other configuration update needed.
+    //      Once you configure this field, it is not safe to update it.
+    //
+    //    - Configure `allowedUnsignedAttributes`: You may also explicitly list
+    //      a set of attributes that should be considered unauthenticated when encountered
+    //      on read. Be careful if you use this configuration. Do not remove an attribute
+    //      name from this configuration, even if you are no longer writing with that attribute,
+    //      as old items may still include this attribute, and our configuration needs to know
+    //      to continue to exclude this attribute from the signature scope.
+    //      If you add new attribute names to this field, you must first deploy the update to this
+    //      field to all readers in your host fleet before deploying the update to start writing
+    //      with that new attribute.
+    //
+    //   For this example, we will explicitly list the attributes that are not signed.
+    let unsigned_attributes = vec!["attribute3".to_string()];
+
+    // Create the DynamoDb Encryption configuration for the table we will be writing to.
+    let table_config = DynamoDbTableEncryptionConfig::builder()
+        .logical_table_name(ddb_table_name)
+        .partition_key_name(partition_key_name)
+        .sort_key_name(sort_key_name)
+        .attribute_actions_on_encrypt(attribute_actions_on_encrypt)
+        .keyring(kms_keyring)
+        .allowed_unsigned_attributes(unsigned_attributes)
+        .plaintext_override(plaintext_override)
+        .build()?;
+
+    let table_configs = DynamoDbTablesEncryptionConfig::builder()
+        .table_encryption_configs(HashMap::from([(ddb_table_name.to_string(), table_config)]))
+        .build()?;
+
+    Ok(table_configs)
+}

DynamoDbEncryption/runtimes/rust/examples/migration/plaintext_to_awsdbe/awsdbe/migration_step_1.rs

@@ -0,0 +1,185 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use aws_sdk_dynamodb::types::AttributeValue;
+use std::collections::HashMap;
+use aws_db_esdk::intercept::DbEsdkInterceptor;
+use aws_db_esdk::dynamodb::types::PlaintextOverride;
+use crate::migration::plaintext_to_awsdbe::migration_utils::{
+    verify_returned_item, ENCRYPTED_AND_SIGNED_VALUE, SIGN_ONLY_VALUE, DO_NOTHING_VALUE,
+};
+use crate::migration::plaintext_to_awsdbe::awsdbe::common::create_table_configs;
+
+/*
+Migration Step 1: This is the first step in the migration process from
+plaintext to encrypted DynamoDB using the AWS Database Encryption SDK.
+
+In this example, we configure a DynamoDB Encryption client to do the following:
+1. Write items only in plaintext
+2. Read items in plaintext or, if the item is encrypted, decrypt with our encryption configuration
+
+While this step configures your client to be ready to start reading encrypted items,
+we do not yet expect to be reading any encrypted items,
+as our client still writes plaintext items.
+Before you move on to step 2, ensure that these changes have successfully been deployed
+to all of your readers.
+
+Running this example requires access to the DDB Table whose name
+is provided in the function parameter.
+This table must be configured with the following
+primary key configuration:
+  - Partition key is named "partition_key" with type (S)
+  - Sort key is named "sort_key" with type (N)
+*/
+pub async fn migration_step_1_example(
+    kms_key_id: &str,
+    ddb_table_name: &str,
+    partition_key_value: &str,
+    sort_key_write_value: &str,
+    sort_key_read_value: &str,
+) -> Result<bool, Box<dyn std::error::Error>> {
+    // 1. Create table configurations
+    // In this step of migration we will use PlaintextOverride::ForcePlaintextWriteAllowPlaintextRead
+    // which means:
+    //     - Write: Items are forced to be written as plaintext.
+    //              Items may not be written as encrypted items.
+    //     - Read: Items are allowed to be read as plaintext.
+    //             Items are allowed to be read as encrypted items.
+    let table_configs = create_table_configs(
+        kms_key_id,
+        ddb_table_name,
+        PlaintextOverride::ForcePlaintextWriteAllowPlaintextRead,
+    )
+    .await?;
+
+    // 2. Create a new AWS SDK DynamoDb client using the TableEncryptionConfigs
+    let sdk_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
+    let dynamo_config = aws_sdk_dynamodb::config::Builder::from(&sdk_config)
+        .interceptor(DbEsdkInterceptor::new(table_configs)?)
+        .build();
+    let ddb = aws_sdk_dynamodb::Client::from_conf(dynamo_config);
+
+    // 3. Put an item into our table using the above client.
+    //    This item will be stored in plaintext due to our PlaintextOverride configuration.
+    let partition_key_name = "partition_key";
+    let sort_key_name = "sort_key";
+    let encrypted_and_signed_value = ENCRYPTED_AND_SIGNED_VALUE;
+    let sign_only_value = SIGN_ONLY_VALUE;
+    let do_nothing_value = DO_NOTHING_VALUE;
+    let item = HashMap::from([
+        (
+            partition_key_name.to_string(),
+            AttributeValue::S(partition_key_value.to_string()),
+        ),
+        (
+            sort_key_name.to_string(),
+            AttributeValue::N(sort_key_write_value.to_string()),
+        ),
+        (
+            "attribute1".to_string(),
+            AttributeValue::S(encrypted_and_signed_value.to_string()),
+        ),
+        (
+            "attribute2".to_string(),
+            AttributeValue::S(sign_only_value.to_string()),
+        ),
+        (
+            "attribute3".to_string(),
+            AttributeValue::S(do_nothing_value.to_string()),
+        ),
+    ]);
+
+    ddb.put_item()
+        .table_name(ddb_table_name)
+        .set_item(Some(item))
+        .send()
+        .await?;
+
+    // 4. Get an item back from the table using the same client.
+    //    If this is an item written in plaintext (i.e. any item written
+    //    during Step 0 or 1), then the item will still be in plaintext.
+    //    If this is an item that was encrypted client-side (i.e. any item written
+    //    during Step 2 or after), then the item will be decrypted client-side
+    //    and surfaced as a plaintext item.
+    let key = HashMap::from([
+        (
+            partition_key_name.to_string(),
+            AttributeValue::S(partition_key_value.to_string()),
+        ),
+        (
+            sort_key_name.to_string(),
+            AttributeValue::N(sort_key_read_value.to_string()),
+        ),
+    ]);
+
+    let response = ddb
+        .get_item()
+        .table_name(ddb_table_name)
+        .set_key(Some(key))
+        // In this example we configure a strongly consistent read
+        // because we perform a read immediately after a write (for demonstrative purposes).
+        // By default, reads are only eventually consistent.
+        .consistent_read(true)
+        .send()
+        .await?;
+
+    // 5. Verify we get the expected item back
+    if let Some(item) = response.item {
+        let success = verify_returned_item(&item, partition_key_value, sort_key_read_value)?;
+        if success {
+            println!("MigrationStep1 completed successfully");
+        }
+        Ok(success)
+    } else {
+        Err("No item found".into())
+    }
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_migration_step_1() -> Result<(), Box<dyn std::error::Error>> {
+    use crate::migration::plaintext_to_awsdbe::plaintext::migration_step_0::migration_step_0_example;
+    use crate::migration::plaintext_to_awsdbe::awsdbe::migration_step_2::migration_step_2_example;
+    use crate::migration::plaintext_to_awsdbe::awsdbe::migration_step_3::migration_step_3_example;
+    use crate::test_utils;
+    use uuid::Uuid;
+
+    let kms_key_id = test_utils::TEST_KMS_KEY_ID;
+    let table_name = test_utils::TEST_DDB_TABLE_NAME;
+    let partition_key = Uuid::new_v4().to_string();
+    let sort_keys = ["0", "1", "2", "3"];
+    
+    // Successfully executes step 1
+    let success = migration_step_1_example(kms_key_id, table_name, &partition_key, sort_keys[1], sort_keys[1]).await?;
+    assert!(success, "MigrationStep1 should complete successfully");
+
+    // Given: Step 0 has succeeded
+    let success = migration_step_0_example(table_name, &partition_key, sort_keys[0], sort_keys[0]).await?;
+    assert!(success, "MigrationStep0 should complete successfully");
+
+    // When: Execute Step 1 with sortReadValue=0, Then: Success (i.e. can read plaintext values from Step 0)
+    let success = migration_step_1_example(kms_key_id, table_name, &partition_key, sort_keys[1], sort_keys[0]).await?;
+    assert!(success, "MigrationStep1 should be able to read items written by Step 0");
+
+    // Given: Step 2 has succeeded
+    let success = migration_step_2_example(kms_key_id, table_name, &partition_key, sort_keys[2], sort_keys[2]).await?;
+    assert!(success, "MigrationStep2 should complete successfully");
+
+    // When: Execute Step 1 with sortReadValue=2, Then: Success (i.e. can read encrypted values from Step 2)
+    let success = migration_step_1_example(kms_key_id, table_name, &partition_key, sort_keys[1], sort_keys[2]).await?;
+    assert!(success, "MigrationStep1 should be able to read items written by Step 2");
+
+    // Given: Step 3 has succeeded
+    let success = migration_step_3_example(kms_key_id, table_name, &partition_key, sort_keys[3], sort_keys[3]).await?;
+    assert!(success, "MigrationStep3 should complete successfully");
+
+    // When: Execute Step 1 with sortReadValue=3, Then: Success (i.e. can read encrypted values from Step 3)
+    let success = migration_step_1_example(kms_key_id, table_name, &partition_key, sort_keys[1], sort_keys[3]).await?;
+    assert!(success, "MigrationStep1 should be able to read items written by Step 3");
+
+    // Cleanup
+    for sort_key in &sort_keys {
+        test_utils::cleanup_items(table_name, &partition_key, sort_key).await?;
+    }
+
+    Ok(())
+}