From bdd6f3fa78db8176d0818349d6738f6cd78044e5 Mon Sep 17 00:00:00 2001 From: Satheesha Chattenahalli Hanume Gowda Date: Tue, 26 Aug 2025 16:29:12 -0700 Subject: [PATCH] Redis/RDB backward downgrade compatibility from Redis 7.2 and Valkey 7.2/8.0 to Redis 7.0 Signed-off-by: Satheesha Chattenahalli Hanume Gowda --- redis.conf | 43 ++++--- src/cluster.c | 5 +- src/config.c | 5 + src/rdb.c | 120 +++++++++++++++++--- src/rdb.h | 22 +++- src/server.h | 6 + tests/assets/encodings-rdb987.rdb | Bin 0 -> 675 bytes tests/assets/set_listpack_fixture.rdb | Bin 0 -> 148 bytes tests/integration/rdb.tcl | 57 +++++++--- tests/integration/rdb_load_set_listpack.tcl | 62 ++++++++++ tests/integration/set_listpack_fixture.rdb | Bin 0 -> 148 bytes tests/support/util.tcl | 22 ++++ tests/test_helper.tcl | 1 + tests/unit/dump.tcl | 17 +++ 14 files changed, 312 insertions(+), 48 deletions(-) create mode 100644 tests/assets/encodings-rdb987.rdb create mode 100644 tests/assets/set_listpack_fixture.rdb create mode 100644 tests/integration/rdb_load_set_listpack.tcl create mode 100644 tests/integration/set_listpack_fixture.rdb diff --git a/redis.conf b/redis.conf index 0431c37428..05dd24027c 100644 --- a/redis.conf +++ b/redis.conf @@ -462,6 +462,15 @@ rdbcompression yes # tell the loading code to skip the check. rdbchecksum yes +# Valkey can try to load an RDB dump produced by a future version of Valkey. +# This can only work on a best-effort basis, because future RDB versions may +# contain information that's not known to the current version. If no new features +# are used, it may be possible to import the data produced by a later version, +# but loading is aborted if unknown information is encountered. Possible values +# are 'strict' and 'relaxed'. This also applies to replication and the RESTORE +# command. +rdb-version-check relaxed + # Enables or disables full sanitization checks for ziplist and listpack etc when # loading an RDB or RESTORE payload. This reduces the chances of a assertion or # crash later on while processing commands. @@ -909,10 +918,10 @@ replica-priority 100 # commands. For instance ~* allows all the keys. The pattern # is a glob-style pattern like the one of KEYS. # It is possible to specify multiple patterns. -# %R~ Add key read pattern that specifies which keys can be read +# %R~ Add key read pattern that specifies which keys can be read # from. # %W~ Add key write pattern that specifies which keys can be -# written to. +# written to. # allkeys Alias for ~* # resetkeys Flush the list of allowed keys patterns. # & Add a glob-style pattern of Pub/Sub channels that can be @@ -939,10 +948,10 @@ replica-priority 100 # -@all. The user returns to the same state it has immediately # after its creation. # () Create a new selector with the options specified within the -# parentheses and attach it to the user. Each option should be -# space separated. The first character must be ( and the last +# parentheses and attach it to the user. Each option should be +# space separated. The first character must be ( and the last # character must be ). -# clearselectors Remove all of the currently attached selectors. +# clearselectors Remove all of the currently attached selectors. # Note this does not change the "root" user permissions, # which are the permissions directly applied onto the # user (outside the parentheses). @@ -968,7 +977,7 @@ replica-priority 100 # Basically ACL rules are processed left-to-right. # # The following is a list of command categories and their meanings: -# * keyspace - Writing or reading from keys, databases, or their metadata +# * keyspace - Writing or reading from keys, databases, or their metadata # in a type agnostic way. Includes DEL, RESTORE, DUMP, RENAME, EXISTS, DBSIZE, # KEYS, EXPIRE, TTL, FLUSHALL, etc. Commands that may modify the keyspace, # key or metadata will also have `write` category. Commands that only read @@ -1589,8 +1598,8 @@ aof-timestamp-enabled no # # cluster-node-timeout 15000 -# The cluster port is the port that the cluster bus will listen for inbound connections on. When set -# to the default value, 0, it will be bound to the command port + 10000. Setting this value requires +# The cluster port is the port that the cluster bus will listen for inbound connections on. When set +# to the default value, 0, it will be bound to the command port + 10000. Setting this value requires # you to specify the cluster bus port when executing cluster meet. # cluster-port 0 @@ -1725,12 +1734,12 @@ aof-timestamp-enabled no # PubSub message by default. (client-query-buffer-limit default value is 1gb) # # cluster-link-sendbuf-limit 0 - -# Clusters can configure their announced hostname using this config. This is a common use case for + +# Clusters can configure their announced hostname using this config. This is a common use case for # applications that need to use TLS Server Name Indication (SNI) or dealing with DNS based # routing. By default this value is only shown as additional metadata in the CLUSTER SLOTS -# command, but can be changed using 'cluster-preferred-endpoint-type' config. This value is -# communicated along the clusterbus to all nodes, setting it to an empty string will remove +# command, but can be changed using 'cluster-preferred-endpoint-type' config. This value is +# communicated along the clusterbus to all nodes, setting it to an empty string will remove # the hostname and also propagate the removal. # # cluster-announce-hostname "" @@ -1739,13 +1748,13 @@ aof-timestamp-enabled no # a user defined hostname, or by declaring they have no endpoint. Which endpoint is # shown as the preferred endpoint is set by using the cluster-preferred-endpoint-type # config with values 'ip', 'hostname', or 'unknown-endpoint'. This value controls how -# the endpoint returned for MOVED/ASKING requests as well as the first field of CLUSTER SLOTS. -# If the preferred endpoint type is set to hostname, but no announced hostname is set, a '?' +# the endpoint returned for MOVED/ASKING requests as well as the first field of CLUSTER SLOTS. +# If the preferred endpoint type is set to hostname, but no announced hostname is set, a '?' # will be returned instead. # # When a cluster advertises itself as having an unknown endpoint, it's indicating that -# the server doesn't know how clients can reach the cluster. This can happen in certain -# networking situations where there are multiple possible routes to the node, and the +# the server doesn't know how clients can reach the cluster. This can happen in certain +# networking situations where there are multiple possible routes to the node, and the # server doesn't know which one the client took. In this case, the server is expecting # the client to reach out on the same endpoint it used for making the last request, but use # the port provided in the response. @@ -2058,7 +2067,7 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # errors or data eviction. To avoid this we can cap the accumulated memory # used by all client connections (all pubsub and normal clients). Once we # reach that limit connections will be dropped by the server freeing up -# memory. The server will attempt to drop the connections using the most +# memory. The server will attempt to drop the connections using the most # memory first. We call this mechanism "client eviction". # # Client eviction is configured using the maxmemory-clients setting as follows: diff --git a/src/cluster.c b/src/cluster.c index 70ede5cb3e..0fee2e9d7a 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5986,7 +5986,10 @@ int verifyDumpPayload(unsigned char *p, size_t len, uint16_t *rdbver_ptr) { if (rdbver_ptr) { *rdbver_ptr = rdbver; } - if (rdbver > RDB_VERSION) return C_ERR; + if ((rdbver >= RDB_FOREIGN_VERSION_MIN && rdbver <= RDB_FOREIGN_VERSION_MAX) || + (rdbver > RDB_VERSION && server.rdb_version_check == RDB_VERSION_CHECK_STRICT)) { + return C_ERR; + } if (server.skip_checksum_validation) return C_OK; diff --git a/src/config.c b/src/config.c index bfb49ef9c0..d8c6ba9ce8 100644 --- a/src/config.c +++ b/src/config.c @@ -159,6 +159,10 @@ configEnum propagation_error_behavior_enum[] = { {NULL, 0} }; +configEnum rdb_version_check_enum[] = {{"strict", RDB_VERSION_CHECK_STRICT}, // strict: Reject future RDB versions. + {"relaxed", RDB_VERSION_CHECK_RELAXED}, // relaxed: Try parsing future RDB versions and fail only when an unknown RDB opcode or type is encountered. + {NULL, 0}}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -3055,6 +3059,7 @@ standardConfig static_configs[] = { createEnumConfig("propagation-error-behavior", NULL, MODIFIABLE_CONFIG, propagation_error_behavior_enum, server.propagation_error_behavior, PROPAGATION_ERR_BEHAVIOR_IGNORE, NULL, NULL), createEnumConfig("shutdown-on-sigint", NULL, MODIFIABLE_CONFIG | MULTI_ARG_CONFIG, shutdown_on_sig_enum, server.shutdown_on_sigint, 0, isValidShutdownOnSigFlags, NULL), createEnumConfig("shutdown-on-sigterm", NULL, MODIFIABLE_CONFIG | MULTI_ARG_CONFIG, shutdown_on_sig_enum, server.shutdown_on_sigterm, 0, isValidShutdownOnSigFlags, NULL), + createEnumConfig("rdb-version-check", NULL, MODIFIABLE_CONFIG, rdb_version_check_enum, server.rdb_version_check, RDB_VERSION_CHECK_RELAXED, NULL, NULL), /* Integer configs */ createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL), diff --git a/src/rdb.c b/src/rdb.c index 710e04feb2..3a8da74f4d 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -37,13 +37,14 @@ #include #include #include +#include #include #include #include #include #include #include - +#include "listpack.h" /* This macro is called when the internal RDB structure is corrupt */ #define rdbReportCorruptRDB(...) rdbReportError(1, __LINE__,__VA_ARGS__) /* This macro is called when RDB read failed (possibly a short read) */ @@ -1720,6 +1721,90 @@ int lpPairsValidateIntegrityAndDups(unsigned char *lp, size_t size, int deep) { return ret; } +/* + * This new function would be responsible for handling the RDB_TYPE_SET_LISTPACK + */ +robj *rdbLoadSetListpackObject(rio *rdb) { + // Step 1: Load the serialized listpack from the RDB stream. + robj *listpack_obj = rdbLoadStringObject(rdb); + if (listpack_obj == NULL) { + serverLog(LL_WARNING, "RDB: Failed to load string object for RDB_TYPE_SET_LISTPACK type."); + return NULL; + } + + if (listpack_obj->type != OBJ_STRING || !sdsEncodedObject(listpack_obj)) { + serverLog(LL_WARNING, "RDB: Loaded object for RDB_TYPE_SET_LISTPACK is not a valid string type."); + decrRefCount(listpack_obj); + return NULL; + } + + sds listpack_sds = sdsdup(listpack_obj->ptr); + decrRefCount(listpack_obj); // We have our own copy of the SDS now. + + if (listpack_sds == NULL) { + serverLog(LL_WARNING, "RDB: OOM when duplicating SDS for RDB_TYPE_SET_LISTPACK."); + return NULL; + } + + // Step 2: Create a new Set object for the older Redis version. + robj *set_obj = createSetObject(); + if (set_obj == NULL) { + serverLog(LL_WARNING, "RDB: Failed to createSetObject for RDB_TYPE_SET_LISTPACK."); + sdsfree(listpack_sds); + return NULL; + } + + // Step 3: Iterate through the listpack and add elements to the Set. + unsigned char *lp = (unsigned char *)listpack_sds; + unsigned char *fptr; + unsigned char *eptr; + unsigned char *vstr; + unsigned int vlen; + long long vlong; + + fptr = lpFirst(lp); + eptr = fptr; + + while (eptr) { + vstr = lpGetValue(eptr, &vlen, &vlong); + sds ele_sds; + + if (vstr) { + ele_sds = sdsnewlen((const void *)vstr, vlen); + } else { + ele_sds = sdsfromlonglong(vlong); + } + + if (ele_sds == NULL) { + serverLog(LL_WARNING, "RDB: OOM creating SDS for set element from RDB_TYPE_SET_LISTPACK ."); + sdsfree(listpack_sds); + decrRefCount(set_obj); + return NULL; + } + + // Add the element to the set. + // setTypeAdd in Redis 7.0.x (and similar versions) does NOT take ownership + // of the 'ele_sds' passed to it. It either uses the integer value directly + // (for intsets) or creates its own sdsdup for hashtables. + // Therefore, 'ele_sds' must always be freed by this function after the call. + setTypeAdd(set_obj, ele_sds); + // We don't strictly need to check the return value of setTypeAdd for freeing ele_sds, + // as it's never consumed by setTypeAdd. However, checking it could be useful + // for logging or other error handling if an element fails to be added for other reasons. + + sdsfree(ele_sds); // Always free ele_sds as it's not consumed by setTypeAdd. + + eptr = lpNext(lp, eptr); + } + + // Step 4: Clean up the duplicated raw listpack SDS. + sdsfree(listpack_sds); + + // Step 5: Return the created and populated set object. + serverLog(LL_VERBOSE, "RDB: Successfully loaded a RDB_TYPE_SET_LISTPACK and converted to OBJ_ENCODING_HT."); + return set_obj; +} + /* Load a Redis object of the specified type from the specified file. * On success a newly allocated object is returned, otherwise NULL. * When the function returns NULL and if 'error' is not NULL, the @@ -1742,8 +1827,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { skip = !!(server.current_client->user->flags & USER_FLAG_SANITIZE_PAYLOAD_SKIP); deep_integrity_validation = !skip; } - - if (rdbtype == RDB_TYPE_STRING) { + if (rdbtype == RDB_TYPE_SET_LISTPACK) { + o = rdbLoadSetListpackObject(rdb); + } else if (rdbtype == RDB_TYPE_STRING) { /* Read string value */ if ((o = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; o = tryObjectEncoding(o); @@ -2317,7 +2403,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { rdbReportCorruptRDB("Unknown RDB encoding type %d",rdbtype); break; } - } else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS || rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { + } else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS || rdbtype == RDB_TYPE_STREAM_LISTPACKS_2 || rdbtype == RDB_TYPE_STREAM_LISTPACKS_3) { o = createStreamObject(); stream *s = o->ptr; uint64_t listpacks = rdbLoadLen(rdb,NULL); @@ -2394,7 +2480,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { s->last_id.ms = rdbLoadLen(rdb,NULL); s->last_id.seq = rdbLoadLen(rdb,NULL); - if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { + if (rdbtype >= RDB_TYPE_STREAM_LISTPACKS_2) { /* Load the first entry ID. */ s->first_id.ms = rdbLoadLen(rdb,NULL); s->first_id.seq = rdbLoadLen(rdb,NULL); @@ -2461,7 +2547,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { /* Load group offset. */ uint64_t cg_offset; - if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { + if (rdbtype >= RDB_TYPE_STREAM_LISTPACKS_2) { cg_offset = rdbLoadLen(rdb,NULL); if (rioGetReadError(rdb)) { rdbReportReadError("Stream cgroup offset loading failed."); @@ -2902,18 +2988,25 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin char buf[1024]; int error; long long empty_keys_skipped = 0; + bool is_valkey_magic; rdb->update_cksum = rdbLoadProgressCallback; rdb->max_processing_chunk = server.loading_process_events_interval_bytes; if (rioRead(rdb,buf,9) == 0) goto eoferr; buf[9] = '\0'; - if (memcmp(buf,"REDIS",5) != 0) { - serverLog(LL_WARNING,"Wrong signature trying to load DB from file"); + if (memcmp(buf, "REDIS0", 6) == 0) { + is_valkey_magic = false; + } else if (memcmp(buf, "VALKEY", 6) == 0) { + is_valkey_magic = true; + } else { + serverLog(LL_WARNING, "Wrong signature trying to load DB from file"); errno = EINVAL; return C_ERR; } - rdbver = atoi(buf+5); - if (rdbver < 1 || rdbver > RDB_VERSION) { + rdbver = atoi(buf + 6); + if (rdbver < 1 || + (rdbver >= RDB_FOREIGN_VERSION_MIN && !is_valkey_magic) || + (rdbver > RDB_VERSION && server.rdb_version_check == RDB_VERSION_CHECK_STRICT)) { serverLog(LL_WARNING,"Can't handle RDB format version %d",rdbver); errno = EINVAL; return C_ERR; @@ -3014,9 +3107,10 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin if (rsi) rsi->repl_offset = strtoll(auxval->ptr,NULL,10); } else if (!strcasecmp(auxkey->ptr,"lua")) { /* Won't load the script back in memory anymore. */ - } else if (!strcasecmp(auxkey->ptr,"redis-ver")) { - serverLog(LL_NOTICE,"Loading RDB produced by version %s", - (char*)auxval->ptr); + } else if (!strcasecmp(auxkey->ptr, "redis-ver")) { + serverLog(LL_NOTICE, "Loading RDB produced by Redis version %s", (char *)auxval->ptr); + } else if (!strcasecmp(auxkey->ptr, "valkey-ver")) { + serverLog(LL_NOTICE, "Loading RDB produced by Valkey version %s", (char *)auxval->ptr); } else if (!strcasecmp(auxkey->ptr,"ctime")) { time_t age = time(NULL)-strtol(auxval->ptr,NULL,10); if (age < 0) age = 0; diff --git a/src/rdb.h b/src/rdb.h index 4f057a252b..c90ee38913 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -37,9 +37,27 @@ #include "server.h" /* The current RDB version. When the format changes in a way that is no longer - * backward compatible this number gets incremented. */ + * backward compatible this number gets incremented. + * + * RDB 11 is the last open-source Redis RDB version, used by Valkey 7.x and 8.x. + * + * RDB 12+ are non-open-source Redis formats. + * + * Next time we bump the Valkey RDB version, use much higher version to avoid + * collisions with non-OSS Redis RDB versions. For example, we could use RDB + * version 90 for Valkey 9.0. + * + * In an RDB file/stream, we also check the magic string REDIS or VALKEY but in + * the DUMP/RESTORE format, there is only the RDB version number and no magic + * string. */ #define RDB_VERSION 10 +/* Reserved range for foreign (unsupported, non-OSS) RDB format. */ +#define RDB_FOREIGN_VERSION_MIN 12 +#define RDB_FOREIGN_VERSION_MAX 79 +static_assert(RDB_VERSION < RDB_FOREIGN_VERSION_MIN || RDB_VERSION > RDB_FOREIGN_VERSION_MAX, + "RDB version in foreign version range"); + /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of * the first byte to interpreter the length: @@ -95,6 +113,8 @@ #define RDB_TYPE_ZSET_LISTPACK 17 #define RDB_TYPE_LIST_QUICKLIST_2 18 #define RDB_TYPE_STREAM_LISTPACKS_2 19 +#define RDB_TYPE_SET_LISTPACK 20 +#define RDB_TYPE_STREAM_LISTPACKS_3 21 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Test if a type is an object type. */ diff --git a/src/server.h b/src/server.h index 82e4db9381..643474d90f 100644 --- a/src/server.h +++ b/src/server.h @@ -590,6 +590,11 @@ typedef enum { CLUSTER_ENDPOINT_TYPE_UNKNOWN_ENDPOINT /* Show NULL or empty */ } cluster_endpoint_type; +typedef enum { + RDB_VERSION_CHECK_STRICT = 0, + RDB_VERSION_CHECK_RELAXED +} rdb_version_check_type; + /* RDB active child save type. */ #define RDB_CHILD_TYPE_NONE 0 #define RDB_CHILD_TYPE_DISK 1 /* RDB is written to disk. */ @@ -1635,6 +1640,7 @@ struct redisServer { int active_defrag_enabled; int sanitize_dump_payload; /* Enables deep sanitization for ziplist and listpack in RDB and RESTORE. */ int skip_checksum_validation; /* Disable checksum validation for RDB and RESTORE payload. */ + int rdb_version_check; /* Try to load RDB produced by a future version. */ int jemalloc_bg_thread; /* Enable jemalloc background thread */ size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */ int active_defrag_threshold_lower; /* minimum percentage of fragmentation to start active defrag */ diff --git a/tests/assets/encodings-rdb987.rdb b/tests/assets/encodings-rdb987.rdb new file mode 100644 index 0000000000000000000000000000000000000000..2357671597724721d10930f7a2ef378febcb7a36 GIT binary patch literal 675 zcmbVKu}UOC5UuKJmvv?Z0|U7O+2r6j40>i@UPmmNhA10%sq zxQj0G8;lKlH4!5}VZXq@54g2@xCPA)8){znRCV{O*YEq+Z=35sSKBLpf#gZ)4jaN4 zkt$)W$P^i4C{;=_ny95FgRHfb@qb1;out`PYk8%;iW(E4wMY~iOi61^iBf1WlRVdw z7bCEF+6e&6NW$*ceX$$=n%coxM)x18ja;>;(GI)F!zUT|;~YC=P3>GA3up*Dh> zD~DV*hFWTx3urug4#DZ|-u=z@LVqtFWR%ln*N<>%N`ej#{jcn`$R87?B_c|N>Ea?ZW- LSCC@Ntg7(>Xe^=_ literal 0 HcmV?d00001 diff --git a/tests/assets/set_listpack_fixture.rdb b/tests/assets/set_listpack_fixture.rdb new file mode 100644 index 0000000000000000000000000000000000000000..0f2faddc28759820ad3f221398d0e430e1b00091 GIT binary patch literal 148 zcmWG?b@2=~FfcUy#Z{J=lbu?rTb5eHYN2PKXZ(w+C^aRsST`xNr1*ftFV^Ie%-qyN zagwGPzc@;ZQ&V(vQ*#ep;1*{1#gUkwrkj*loO*!aAH#1(1`)yB%Hq_L_?&|HlGNgo zc%ZRLN(>APtPHJ*IRzPsY%NKtC5f!9>4~|yiEP!3&BeJnEdT!pPs}R&_b1^d0F_NQ A)Bpeg literal 0 HcmV?d00001 diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 104d372e14..43e7f4020a 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -1,9 +1,21 @@ tags {"rdb external:skip"} { +# Helper function to start a server and kill it, just to check the error +# logged. +set defaults {} +proc start_server_and_kill_it {overrides code} { + upvar defaults defaults srv srv server_path server_path + set config [concat $defaults $overrides] + set srv [start_server [list overrides $config keep_persistence true]] + uplevel 1 $code + kill_server $srv +} + set server_path [tmpdir "server.rdb-encoding-test"] # Copy RDB with different encodings in server path exec cp tests/assets/encodings.rdb $server_path +exec cp tests/assets/encodings-rdb987.rdb $server_path exec cp tests/assets/list-quicklist.rdb $server_path start_server [list overrides [list "dir" $server_path "dbfilename" "list-quicklist.rdb"]] { @@ -14,11 +26,7 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "list-quickli } {7} } -start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] { - test "RDB encoding loading test" { - r select 0 - csvdump r - } {"0","compressible","string","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +set csv_dump {"0","compressible","string","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "0","hash","hash","a","1","aa","10","aaa","100","b","2","bb","20","bbb","200","c","3","cc","30","ccc","300","ddd","400","eee","5000000000", "0","hash_zipped","hash","a","1","b","2","c","3", "0","list","list","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000", @@ -32,6 +40,34 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rd "0","zset","zset","a","1","b","2","c","3","aa","10","bb","20","cc","30","aaa","100","bbb","200","ccc","300","aaaa","1000","cccc","123456789","bbbb","5000000000", "0","zset_zipped","zset","a","1","b","2","c","3", } + +start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] { + test "RDB encoding loading test" { + r select 0 + csvdump r + } $csv_dump +} + +start_server_and_kill_it [list "dir" $server_path \ + "dbfilename" "encodings-rdb987.rdb" \ + "rdb-version-check" "strict"] { + test "RDB future version loading, strict version check" { + wait_for_condition 50 100 { + [string match {*Fatal error loading*} \ + [exec tail -1 < [dict get $srv stdout]]] + } else { + fail "Server started even if RDB version check failed" + } + } +} + +start_server [list overrides [list "dir" $server_path \ + "dbfilename" "encodings-rdb987.rdb" \ + "rdb-version-check" "relaxed"]] { + test "RDB future version loading, relaxed version check" { + r select 0 + csvdump r + } $csv_dump } set server_path [tmpdir "server.rdb-startup-test"] @@ -79,17 +115,6 @@ start_server [list overrides [list "dir" $server_path] keep_persistence true] { r del stream } -# Helper function to start a server and kill it, just to check the error -# logged. -set defaults {} -proc start_server_and_kill_it {overrides code} { - upvar defaults defaults srv srv server_path server_path - set config [concat $defaults $overrides] - set srv [start_server [list overrides $config keep_persistence true]] - uplevel 1 $code - kill_server $srv -} - # Make the RDB file unreadable file attributes [file join $server_path dump.rdb] -permissions 0222 diff --git a/tests/integration/rdb_load_set_listpack.tcl b/tests/integration/rdb_load_set_listpack.tcl new file mode 100644 index 0000000000..37d6f4bfa9 --- /dev/null +++ b/tests/integration/rdb_load_set_listpack.tcl @@ -0,0 +1,62 @@ +# This test suite requires a fixture RDB file named "set_listpack_fixture.rdb". +# This fixture must contain a Redis Set object (e.g., key "myset_lp_test_key") +# that was saved using the RDB_TYPE_SET_LISTPACK encoding by a newer Redis version (e.g., 7.2+). +# +# Example to generate set_listpack_fixture.rdb: +# 1. In a Redis 7.2+ CLI: +# SADD myset_lp_test_key "alpha" "beta" "gamma" "123" "sml" +# SAVE +# 2. Copy the generated dump.rdb to "set_listpack_fixture.rdb" in the test directory. + +tags {"rdb external:skip"} { + +# Copy RDB with listpack encoded set to server path +set server_path [tmpdir "server.rdb-encoding-listpack-test"] +exec cp -f tests/assets/set_listpack_fixture.rdb $server_path + +start_server [list overrides [list "dir" $server_path "dbfilename" "set_listpack_fixture.rdb" "rdb-version-check" "relaxed"]] { + + # Test case: Load an RDB file containing a set encoded as RDB_TYPE_SET_LISTPACK. + # This tests the server's ability to correctly parse RDB_TYPE_SET_LISTPACK + # and convert it into its native set representation (e.g., hashtable or intset). + test "Patched Redis should successfully load RDB_TYPE_SET_LISTPACK" { + set test_key_name "myset_lp_test_key" + set expected_elements {"alpha" "beta" "gamma" "123" "sml"} + set expected_cardinality [llength $expected_elements] + + r select 0 + + # 1. Verify the key NOW EXISTS because the patch should handle it. + assert_equal 1 [r exists $test_key_name] "Key '$test_key_name' SHOULD exist after loading" + + # 2. Verify the set's cardinality. + assert_equal $expected_cardinality [r scard $test_key_name] "Cardinality mismatch for '$test_key_name' with patched Redis" + + # 3. Verify individual members. + foreach element $expected_elements { + assert_equal 1 [r sismember $test_key_name $element] "Element '$element' missing from '$test_key_name' with patched Redis" + } + assert_equal 0 [r sismember $test_key_name "nonexistent_element"] "Non-existent element check failed for '$test_key_name' with patched Redis" + + # 4. Verify all members using SMEMBERS. + set loaded_members [lsort [r smembers $test_key_name]] + set expected_members_sorted [lsort $expected_elements] + assert_equal $expected_members_sorted $loaded_members "SMEMBERS content mismatch for '$test_key_name' with patched Redis" + + # 5. Verify the internal encoding of the loaded set. + # After being loaded by the (patched) older Redis, the set should now be + # in one of the older Redis's native encodings (e.g., "hashtable" or "intset"). + # For the mixed elements "alpha", "123", etc., it should be "hashtable". + set encoding_info [r debug object $test_key_name] + assert_match {*encoding:hashtable*} $encoding_info "Set '$test_key_name' encoding is not 'hashtable' after RDB load by patched Redis. Actual: $encoding_info" + + # 6. Ensure no critical errors were logged regarding this type (optional, but good). + # This is the opposite of the incompatibility test. We expect no "Unknown RDB type 18" error. + # This might require a helper like `assert_log_does_not_contain` or checking log length. + # For simplicity, this explicit check is omitted here, but successful loading implies no fatal RDB error. + + # Clean up the key for subsequent tests if any. + r del $test_key_name + } +} +} \ No newline at end of file diff --git a/tests/integration/set_listpack_fixture.rdb b/tests/integration/set_listpack_fixture.rdb new file mode 100644 index 0000000000000000000000000000000000000000..0f2faddc28759820ad3f221398d0e430e1b00091 GIT binary patch literal 148 zcmWG?b@2=~FfcUy#Z{J=lbu?rTb5eHYN2PKXZ(w+C^aRsST`xNr1*ftFV^Ie%-qyN zagwGPzc@;ZQ&V(vQ*#ep;1*{1#gUkwrkj*loO*!aAH#1(1`)yB%Hq_L_?&|HlGNgo zc%ZRLN(>APtPHJ*IRzPsY%NKtC5f!9>4~|yiEP!3&BeJnEdT!pPs}R&_b1^d0F_NQ A)Bpeg literal 0 HcmV?d00001 diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 891ce6730a..a16382d668 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -1074,3 +1074,25 @@ proc memory_usage {key} { } return $usage } + +# Breakpoint function, which invokes a minimal debugger. +# This function can be placed within the desired Tcl tests for debugging purposes. +# +# Arguments: +# * 's': breakpoint label, which is printed when breakpoints are hit for unique identification. +# +# Source: https://wiki.tcl-lang.org/page/A+minimal+debugger +proc bp {{s {}}} { + if ![info exists ::bp_skip] { + set ::bp_skip [list] + } elseif {[lsearch -exact $::bp_skip $s]>=0} return + if [catch {info level -1} who] {set who ::} + while 1 { + puts -nonewline "$who/$s> "; flush stdout + gets stdin line + if {$line=="c"} {puts "continuing.."; break} + if {$line=="i"} {set line "info locals"} + catch {uplevel 1 $line} res + puts $res + } +} \ No newline at end of file diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index aeeabcde49..8f399892eb 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -52,6 +52,7 @@ set ::all_tests { integration/aof integration/aof-multi-part integration/rdb + integration/rdb_load_set_listpack integration/corrupt-dump integration/corrupt-dump-fuzzer integration/convert-zipmap-hash-on-load diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index 2e940bda77..7510ed573c 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -96,6 +96,23 @@ start_server {tags {"dump"}} { set e } {*syntax*} + test {RESTORE key with future RDB version, strict version check} { + r config set rdb-version-check strict + # str len "bar" RDB 222 CRC64 checksum + # | | | | | + set bar_dump "\x00\x03\x62\x61\x72\xde\x00\x0fYUza\xd3\xec\xe0" + assert_error {ERR DUMP payload version or checksum are wrong} {r restore foo 0 $bar_dump replace} + } + + test {RESTORE key with future RDB version, relaxed version check} { + r config set rdb-version-check relaxed + # |type|len| | RDB | CRC64 | + # |str | 3 | "bar" | 222 | checksum | + r restore foo 0 "\x00\x03\x62\x61\x72\xde\x00\x0fYUza\xd3\xec\xe0" replace + r config set rdb-version-check strict + assert_equal {bar} [r get foo] + } + test {DUMP of non existing key returns nil} { r dump nonexisting_key } {}