diff --git a/BUILD.bazel b/BUILD.bazel index 23403fe085a..384a1abe174 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -140,6 +140,7 @@ xnnpack_cc_library( "include/xnnpack.h", ], deps = [ + ":fingerprint_id", "@pthreadpool", ], ) @@ -773,6 +774,54 @@ xnnpack_cc_library( ], ) +xnnpack_cc_library( + name = "fingerprint_id", + srcs = ["src/operators/fingerprint_id.c"], + hdrs = [ + "src/operators/fingerprint_id.h", + "src/operators/fingerprint_id.h.inc", + ], +) + +xnnpack_cc_library( + name = "fingerprint_cache", + srcs = ["src/operators/fingerprint_cache.c"], + hdrs = ["src/operators/fingerprint_cache.h"], + deps = [ + ":allocator", + ":cache", + ":fingerprint_id", + ":init_once", + ":mutex", + ":operator_delete", + ":xnnpack_h", + ], +) + +xnnpack_cc_library( + name = "fingerprint_check", + srcs = ["src/xnnpack/fingerprint_check.c"], + deps = [ + ":fingerprint_cache", + ":fingerprint_id", + ":operators", + ":xnnpack_h", + ], +) + +xnnpack_cc_library( + name = "operator_delete", + srcs = ["src/operator-delete.c"], + deps = [ + ":allocator", + ":logging", + ":operator_h", + ":operator_utils", + ":params", + ":xnnpack_h", + ], +) + xnnpack_cc_library( name = "operators", srcs = OPERATOR_SRCS, @@ -804,6 +853,7 @@ xnnpack_cc_library( ":microparams_init", ":node_type", ":normalization", + ":operator_delete", ":operator_type", ":operator_utils", ":pack_lh", @@ -932,6 +982,7 @@ xnnpack_cc_library( "//:allocator", "//:build_identifier", # build_cleaner: keep "//:common", # build_cleaner: keep + "//:fingerprint_id", "//:init_once", "//:logging", "//:math", # build_cleaner: keep diff --git a/CMakeLists.txt b/CMakeLists.txt index 55e98181b75..ce3ce87686e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -939,6 +939,10 @@ IF(XNNPACK_BUILD_LIBRARY) ADD_LIBRARY(xnnpack-allocator OBJECT src/allocator.c) ADD_LIBRARY(xnnpack-cache OBJECT src/cache.c) ADD_LIBRARY(xnnpack-datatype OBJECT src/datatype.c) + ADD_LIBRARY(xnnpack-operator-delete OBJECT src/operator-delete.c) + ADD_LIBRARY(xnnpack-fingerprint-id OBJECT src/operators/fingerprint_id.c) + ADD_LIBRARY(xnnpack-fingerprint-cache OBJECT src/operators/fingerprint_cache.c) + ADD_LIBRARY(xnnpack-fingerprint-check OBJECT src/xnnpack/fingerprint_check.c) ADD_LIBRARY(xnnpack-memory OBJECT src/memory.c) ADD_LIBRARY(xnnpack-microkernel-utils OBJECT src/microkernel-utils.c) ADD_LIBRARY(xnnpack-mutex OBJECT src/mutex.c) @@ -962,21 +966,25 @@ IF(XNNPACK_BUILD_LIBRARY) TARGET_LINK_LIBRARIES(xnnpack-allocator PRIVATE xnnpack-base xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-cache PRIVATE xnnpack-base xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-datatype PRIVATE xnnpack-base) + TARGET_LINK_LIBRARIES(xnnpack-operator-delete PRIVATE xnnpack-base xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-memory PRIVATE xnnpack-base xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-microkernel-utils PRIVATE xnnpack-base xnnpack-hardware-config xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-mutex PRIVATE xnnpack-base xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-operators PRIVATE xnnpack-base xnnpack-allocator xnnpack-indirection xnnpack-logging xnnpack-microkernel-utils xnnpack-normalization xnnpack-operator-utils xnnpack-pack-lh xnnpack-packing xnnpack-reference-ukernels xnnpack-datatype) + TARGET_LINK_LIBRARIES(xnnpack-fingerprint-cache PRIVATE xnnpack-base xnnpack-allocator xnnpack-cache xnnpack-mutex xnnpack-operator-delete) + TARGET_LINK_LIBRARIES(xnnpack-fingerprint-check PRIVATE xnnpack-base xnnpack-fingerprint-cache xnnpack-operators) TARGET_LINK_LIBRARIES(xnnpack-operator-run PRIVATE xnnpack-base xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-operator-utils PRIVATE xnnpack-base xnnpack-logging) TARGET_LINK_LIBRARIES(xnnpack-reference-ukernels PRIVATE xnnpack-base xnnpack-datatype) TARGET_LINK_LIBRARIES(xnnpack-subgraph PRIVATE xnnpack-base xnnpack-allocator xnnpack-logging xnnpack-memory xnnpack-mutex xnnpack-operators xnnpack-operator-run xnnpack-datatype) TARGET_LINK_LIBRARIES(XNNPACK PRIVATE xnnpack-base xnnpack-allocator xnnpack-cache xnnpack-hardware-config xnnpack-indirection xnnpack-memory xnnpack-microkernel-utils xnnpack-microparams-init - xnnpack-mutex xnnpack-normalization xnnpack-operators xnnpack-operator-run xnnpack-operator-utils xnnpack-pack-lh xnnpack-packing - xnnpack-microkernels-prod xnnpack-subgraph xnnpack-datatype xnnpack-reference-ukernels) - TARGET_LINK_LIBRARIES(XNNPACK PUBLIC pthreadpool xnnpack-logging) + xnnpack-mutex xnnpack-normalization xnnpack-operators xnnpack-operator-run + xnnpack-operator-utils xnnpack-pack-lh xnnpack-packing xnnpack-fingerprint-id xnnpack-fingerprint-cache xnnpack-microkernels-prod + xnnpack-subgraph xnnpack-datatype xnnpack-reference-ukernels) + TARGET_LINK_LIBRARIES(XNNPACK PUBLIC pthreadpool xnnpack-logging xnnpack-fingerprint-check) SET_TARGET_PROPERTIES(XNNPACK PROPERTIES C_EXTENSIONS YES) ENDIF() IF(NOT MSVC) diff --git a/build_srcs.bzl b/build_srcs.bzl index ea6b571f37f..8109b5d48ad 100644 --- a/build_srcs.bzl +++ b/build_srcs.bzl @@ -12,7 +12,6 @@ Lists of target-specific sources used to build XNNPACK. """ OPERATOR_SRCS = [ - "src/operator-delete.c", "src/operator-run.c", "src/operators/argmax-pooling-nhwc.c", "src/operators/average-pooling-nhwc.c", diff --git a/include/experimental.h b/include/experimental.h index c2d2bf61fa6..74d36bb69bb 100644 --- a/include/experimental.h +++ b/include/experimental.h @@ -16,6 +16,7 @@ #include #include "include/xnnpack.h" +#include "src/operators/fingerprint_id.h" #ifdef __cplusplus extern "C" { @@ -98,6 +99,29 @@ enum xnn_status xnn_update_runtime_with_threadpool( xnn_runtime_t runtime, xnn_threadpool_t threadpool); + +typedef struct xnn_fingerprint* xnn_fingerprint_t; + +struct xnn_fingerprint { + enum xnn_fingerprint_id id; + uint32_t value; +}; + +/// Check whether the given configuration matches one that is currently in use. +/// +/// @returns True if the configuration matches. +bool xnn_check_fingerprint(struct xnn_fingerprint fingerprint); + +/// Return the fingerprint corresponding to the given id or NULL if it wasn't +/// set. +const struct xnn_fingerprint* xnn_get_fingerprint(enum xnn_fingerprint_id id); + +/// Set the given fingerprint. +void xnn_set_fingerprint(struct xnn_fingerprint fingerprint); + +/// Clear all fingerprints that were computed until now. +void xnn_clear_fingerprints(); + #ifdef __cplusplus } // extern "C" #endif diff --git a/include/xnnpack.h b/include/xnnpack.h index 25da2891f60..3db3065b7bd 100644 --- a/include/xnnpack.h +++ b/include/xnnpack.h @@ -1,7 +1,7 @@ // Copyright (c) Facebook, Inc. and its affiliates. // All rights reserved. // -// Copyright 2019 Google LLC +// Copyright 2019-2025 Google LLC // // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. @@ -15,6 +15,7 @@ #include #include +#include "src/operators/fingerprint_id.h" #include #ifdef __cplusplus @@ -2297,6 +2298,8 @@ struct xnn_weights_cache_look_up_key { const void* kernel; /// Pointer to the original bias, could be NULL. const void* bias; + + enum xnn_fingerprint_id fingerprint_id; }; /// A group of function pointers to manage weights cache. All functions may be diff --git a/src/operators/fingerprint_cache.c b/src/operators/fingerprint_cache.c new file mode 100644 index 00000000000..8a751169ba4 --- /dev/null +++ b/src/operators/fingerprint_cache.c @@ -0,0 +1,178 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "src/operators/fingerprint_cache.h" + +#include +#include +#include +#include + +#include "include/experimental.h" +#include "include/xnnpack.h" +#include "src/operators/fingerprint_id.h" +#include "src/xnnpack/allocator.h" +#include "src/xnnpack/cache.h" +#include "src/xnnpack/init-once.h" +#include "src/xnnpack/mutex.h" + +#define XNN_FINGERPRINT_MAX_COUNT 256 +static struct xnn_fingerprint fingerprint_vector[XNN_FINGERPRINT_MAX_COUNT]; +static uint32_t fingerprint_vector_size = 0; + +static struct xnn_mutex mutex; +XNN_INIT_ONCE_GUARD(mutex); + +static void init_mutex_config() { xnn_mutex_init(&mutex); } + +const struct xnn_fingerprint* xnn_get_fingerprint( + const enum xnn_fingerprint_id id) { + XNN_INIT_ONCE(mutex); + uint32_t i = 0; + xnn_mutex_lock(&mutex); + for (; i < fingerprint_vector_size; ++i) { + if (fingerprint_vector[i].id == id) { + break; + } + } + xnn_mutex_unlock(&mutex); + return i < fingerprint_vector_size ? fingerprint_vector + i : NULL; +} + +void xnn_set_fingerprint(const struct xnn_fingerprint fingerprint) { + XNN_INIT_ONCE(mutex); + uint32_t i = 0; + xnn_mutex_lock(&mutex); + for (; i < fingerprint_vector_size; ++i) { + if (fingerprint_vector[i].id == fingerprint.id) { + fingerprint_vector[i] = fingerprint; + } + } + xnn_mutex_unlock(&mutex); + if (i < fingerprint_vector_size) { + return; + } + assert(fingerprint_vector_size < XNN_FINGERPRINT_MAX_COUNT); + fingerprint_vector[fingerprint_vector_size++] = fingerprint; +} + +void xnn_clear_fingerprints() { + XNN_INIT_ONCE(mutex); + xnn_mutex_lock(&mutex); + fingerprint_vector_size = 0; + xnn_mutex_unlock(&mutex); +} + +// The context for an XNNPack weight cache provider that we pass to operator +// `create` functions when we want to fingerprint them. +struct fingerprint_cache_context { + void* buffer; + size_t bytes; + uint32_t hash; +}; + +static size_t fingerprint_cache_look_up( + void* context, const struct xnn_weights_cache_look_up_key* cache_key) { + return XNN_CACHE_NOT_FOUND; +} + +static void* fingerprint_cache_reserve_space(void* const context, size_t n) { + struct fingerprint_cache_context* const ctx = context; + assert(ctx); + if (ctx->buffer && ctx->bytes < n) { + xnn_release_simd_memory(ctx->buffer); + ctx->buffer = NULL; + } + if (ctx->buffer == NULL) { + ctx->buffer = xnn_allocate_simd_memory(n); + ctx->bytes = ctx->buffer ? n : 0; + } + return ctx->buffer; +} + +static size_t fingerprint_cache_look_up_or_insert( + void* context, const struct xnn_weights_cache_look_up_key* cache_key, + void* ptr, size_t size) { + assert(context); + struct fingerprint_cache_context* const ctx = context; + ctx->hash = murmur_hash3(ptr, size, /*seed=*/ctx->hash); + return 0; +} + +static bool fingerprint_cache_is_finalized(void* context) { return false; } + +static void* fingerprint_cache_offset_to_addr(void* context, size_t offset) { + assert(context); + struct fingerprint_cache_context* const ctx = context; + return ctx->buffer; +} + +static enum xnn_status fingerprint_cache_delete_cache(void* context) { + struct fingerprint_cache_context* const ctx = context; + if (ctx) { + if (ctx->buffer) { + xnn_release_simd_memory(ctx->buffer); + } + *ctx = (struct fingerprint_cache_context){0}; + } + return xnn_status_success; +} + +struct fingerprint_context create_fingerprint_context( + const enum xnn_fingerprint_id fingerprint_id) { + struct fingerprint_context context = { + .status = xnn_status_uninitialized, + .fingerprint_id = fingerprint_id, + .cache = + (struct xnn_weights_cache_provider){ + .context = NULL, + .look_up = fingerprint_cache_look_up, + .reserve_space = fingerprint_cache_reserve_space, + .look_up_or_insert = fingerprint_cache_look_up_or_insert, + .is_finalized = fingerprint_cache_is_finalized, + .offset_to_addr = fingerprint_cache_offset_to_addr, + .delete_cache = fingerprint_cache_delete_cache}, + .op = NULL, + }; + if (context.fingerprint_id == xnn_fingerprint_id_unknown) { + context.status = xnn_status_unsupported_parameter; + } else if (xnn_get_fingerprint(context.fingerprint_id)) { + context.status = xnn_status_success; + } else { + // Do this after the checks to avoid a memory allocation when unnecessary. + context.cache.context = xnn_allocate_zero_memory(sizeof(struct fingerprint_cache_context)); + } + return context; +} + +static void free_fingerprint_cache_provider( + struct xnn_weights_cache_provider* const provider) { + if (provider) { + provider->delete_cache(provider->context); + if (provider->context) { + xnn_release_memory(provider->context); + } + } +} + +void finalize_fingerprint_context(struct fingerprint_context* const context) { + assert(context); + if (context->status == xnn_status_uninitialized) { + xnn_set_fingerprint((struct xnn_fingerprint){ + .id = context->fingerprint_id, + .value = fingerprint_cache_get_fingerprint(&context->cache)}); + } + free_fingerprint_cache_provider(&context->cache); + if (context->op) { + xnn_delete_operator(context->op); + } +} + +uint32_t fingerprint_cache_get_fingerprint( + const struct xnn_weights_cache_provider* const provider) { + assert(provider); + assert(provider->context); + return ((struct fingerprint_cache_context*)provider->context)->hash; +} diff --git a/src/operators/fingerprint_cache.h b/src/operators/fingerprint_cache.h new file mode 100644 index 00000000000..e930fc22054 --- /dev/null +++ b/src/operators/fingerprint_cache.h @@ -0,0 +1,48 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#ifndef XNNPACK_SRC_OPERATORS_FINGERPRINT_CACHE_H_ +#define XNNPACK_SRC_OPERATORS_FINGERPRINT_CACHE_H_ + +#include +#include +#include +#include + +#include "include/xnnpack.h" +#include "src/operators/fingerprint_id.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +struct fingerprint_context { + enum xnn_status status; + enum xnn_fingerprint_id fingerprint_id; + struct xnn_weights_cache_provider cache; + xnn_operator_t op; +}; + +// Creates a fingerprinting context with an explicit fingerprint id. +struct fingerprint_context create_fingerprint_context( + enum xnn_fingerprint_id fingerprint_id); + +// Releases the fingerprinting resources. +// +// If the context status is uninitialized, a new fingerprinting entry is created +// for the context's `fingerprint_id` and the value computed by +// `fingerprint_cache_get_fingerprint(&context->cache)`. +void finalize_fingerprint_context(struct fingerprint_context* context); + +// Retrieves the fingerprint value from a fingerprinting cache that was created +// by `create_fingerprint_context`. +uint32_t fingerprint_cache_get_fingerprint( + const struct xnn_weights_cache_provider* provider); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // XNNPACK_SRC_OPERATORS_FINGERPRINT_CACHE_H_ diff --git a/src/operators/fingerprint_id.c b/src/operators/fingerprint_id.c new file mode 100644 index 00000000000..a83f0c266e7 --- /dev/null +++ b/src/operators/fingerprint_id.c @@ -0,0 +1,26 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "src/operators/fingerprint_id.h" + +#include +#include + +enum xnn_fingerprint_id xnn_compute_fingerprint_id_value( + enum xnn_fingerprint_id_helper op, enum xnn_fingerprint_id_helper in, + enum xnn_fingerprint_id_helper out, enum xnn_fingerprint_id_helper weights, + ...) { + uint32_t id = op << XNN_FINGERPRINT_ID_OP_OFFSET | + in << XNN_FINGERPRINT_ID_IN_OFFSET | + out << XNN_FINGERPRINT_ID_OUT_OFFSET | + weights << XNN_FINGERPRINT_ID_WEIGHTS_OFFSET; + va_list args; + va_start(args, weights); + enum xnn_fingerprint_id_helper flag; + while ((flag = (enum xnn_fingerprint_id_helper)va_arg(args, int)) != 0) { + id |= flag; + } + return (enum xnn_fingerprint_id)id; +} diff --git a/src/operators/fingerprint_id.h b/src/operators/fingerprint_id.h new file mode 100644 index 00000000000..a9e0c9a5376 --- /dev/null +++ b/src/operators/fingerprint_id.h @@ -0,0 +1,192 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#ifndef XNNPACK_SRC_OPERATORS_FINGERPRINT_ID_H_ +#define XNNPACK_SRC_OPERATORS_FINGERPRINT_ID_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#define XNN_EXPAND(x) x +#define XNN_GLUE(x, y) x y + +#define XNN_COUNT_ARGS_HELPER(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, ...) _9 +#define XNN_COUNT_ARGS_EXPAND(args) XNN_COUNT_ARGS_HELPER args +#define XNN_COUNT_ARGS(...) \ + XNN_COUNT_ARGS_EXPAND((__VA_ARGS__, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)) + +#define XNN_CONCAT_FLAGS_4(prefix, a, b, c, d) \ + prefix##a | prefix##b | prefix##c | prefix##d + +#define XNN_CONCAT_FLAGS_5(prefix, a, b, c, d, e) \ + prefix##a | prefix##b | prefix##c | prefix##d | prefix##e + +#define XNN_CONCAT_FLAGS_6(prefix, a, b, c, d, e, f) \ + prefix##a | prefix##b | prefix##c | prefix##d | prefix##e | prefix##f + +#define XNN_CONCAT_FLAGS_7(prefix, a, b, c, d, e, f, g) \ + prefix##a | prefix##b | prefix##c | prefix##d | prefix##e | prefix##f | \ + prefix##g + +#define XNN_CONCAT_FLAGS_8(prefix, a, b, c, d, e, f, g, h) \ + prefix##a | prefix##b | prefix##c | prefix##d | prefix##e | prefix##f | \ + prefix##g | prefix##h + +#define XNN_CONCAT_FLAGS_9(prefix, a, b, c, d, e, f, g, h, i) \ + prefix##a | prefix##b | prefix##c | prefix##d | prefix##e | prefix##f | \ + prefix##g | prefix##h | prefix##i + +#define XNN_CONCAT_1(a) a +#define XNN_CONCAT_2(a, b) a##_##b +#define XNN_CONCAT_3(a, b, c) a##_##b##_##c +#define XNN_CONCAT_4(a, b, c, d) a##_##b##_##c##_##d +#define XNN_CONCAT_5(a, b, c, d, e) a##_##b##_##c##_##d##_##e +#define XNN_CONCAT_6(a, b, c, d, e, f) a##_##b##_##c##_##d##_##e##_##f +#define XNN_CONCAT_7(a, b, c, d, e, f, g) a##_##b##_##c##_##d##_##e##_##f##_##g +#define XNN_CONCAT_8(a, b, c, d, e, f, g, h) \ + a##_##b##_##c##_##d##_##e##_##f##_##g##_##h +#define XNN_CONCAT_9(a, b, c, d, e, f, g, h, i) \ + a##_##b##_##c##_##d##_##e##_##f##_##g##_##h##_##i + +#define XNN_OVERLOAD_2(a, b) a##b +#define XNN_OVERLOAD_1(a, b) XNN_OVERLOAD_2(a, b) +#define XNN_OVERLOAD(X, C) XNN_OVERLOAD_1(X, C) + +#define XNN_FINGERPRINT_ID_OP_OFFSET 26 +#define XNN_FINGERPRINT_ID_IN_OFFSET 20 +#define XNN_FINGERPRINT_ID_OUT_OFFSET 14 +#define XNN_FINGERPRINT_ID_WEIGHTS_OFFSET 8 + +// Creates a fingerprint id value. +// +// ```cpp +// XNN_FINGERPRINT_ID_VALUE(operator, in_type, out_type, weights_type +// [, flags...]) +// ``` +// +// A fingerprint id value is built using 32 bits. +// +// op in out weights flags +// ├─────┼─────┼─────┼─────┼───────┤ +// 32 26 20 14 8 0 +// +// - operator: 6 bits -> 64 operators possible +// - input, output, weights: 6 bits -> 64 types possible +// - flags: 8 bits -> 8 flags possible +// +// These values should be used with the `XNN_DEFINE_FINGERPRINT_VALUES()` macro +// without the `xnn_fingerprint_id_helper_` prefix. +// +// Example: +// ```cpp +// xnn_fingerprint_id_fully_connected_nc_f16 = +// XNN_FINGERPRINT_ID_VALUE(fully_connected_nc, f16, f16, f16), +// ``` +#define XNN_FINGERPRINT_ID_VALUE(...) \ + XNN_FINGERPRINT_ID_VALUE_EXPAND((__VA_ARGS__)) + +#define XNN_FINGERPRINT_ID_VALUE_EXPAND(args) XNN_FINGERPRINT_ID_VALUE_IMPL args + +#define XNN_FINGERPRINT_ID_VALUE_IMPL(op, in, out, weights, ...) \ + XNN_EXPAND(XNN_GLUE( \ + XNN_OVERLOAD(XNN_CONCAT_FLAGS_, \ + XNN_COUNT_ARGS(op, in, out, weights, __VA_ARGS__)), \ + ((uint32_t)xnn_fingerprint_id_helper_, \ + op << XNN_FINGERPRINT_ID_OP_OFFSET, in << XNN_FINGERPRINT_ID_IN_OFFSET, \ + out << XNN_FINGERPRINT_ID_OUT_OFFSET, \ + weights << XNN_FINGERPRINT_ID_WEIGHTS_OFFSET, __VA_ARGS__))) + +#define XNN_EXPAND_TYPES(prefix, ...) \ + XNN_EXPAND(XNN_GLUE( \ + XNN_OVERLOAD(XNN_EXPAND_TYPES_IMPL_, XNN_COUNT_ARGS(__VA_ARGS__)), \ + (prefix, __VA_ARGS__))) + +#define XNN_EXPAND_TYPES_IMPL_1(prefix, a) XNN_CONCAT_4(prefix, a, a, a) +#define XNN_EXPAND_TYPES_IMPL_2(prefix, a, b) XNN_CONCAT_4(prefix, a, a, b) +#define XNN_EXPAND_TYPES_IMPL_3(prefix, a, b, c) XNN_CONCAT_4(prefix, a, b, c) + +#define XNN_CONCAT_TYPES(prefix, ...) \ + XNN_EXPAND( \ + XNN_GLUE(XNN_OVERLOAD(XNN_CONCAT_, XNN_COUNT_ARGS(prefix, __VA_ARGS__)), \ + (prefix, __VA_ARGS__))) + +#define XNN_FINGERPRINT_ID_NAME(...) \ + XNN_CONCAT_TYPES(xnn_fingerprint_id, __VA_ARGS__) + +// Used to build fingerprint ids. +// +// Because fingerprints may be stored to be compared to past/future versions +// of XNNPack, **these values MUST NOT CHANGE**. You can add new values. +enum xnn_fingerprint_id_helper { + // Operator values + xnn_fingerprint_id_helper_unknown = 0, + xnn_fingerprint_id_helper_test = 1, + xnn_fingerprint_id_helper_no_fingerprint = 2, + // Type values + xnn_fingerprint_id_helper_bf16 = 0, + xnn_fingerprint_id_helper_f16 = 1, + xnn_fingerprint_id_helper_f32 = 2, + xnn_fingerprint_id_helper_pf16 = 3, + xnn_fingerprint_id_helper_pf32 = 4, + xnn_fingerprint_id_helper_qb4w = 5, + xnn_fingerprint_id_helper_qc4w = 6, + xnn_fingerprint_id_helper_qc8w = 7, + xnn_fingerprint_id_helper_qd8 = 8, + xnn_fingerprint_id_helper_qdu8 = 9, + xnn_fingerprint_id_helper_qp8 = 10, + xnn_fingerprint_id_helper_qs8 = 11, + xnn_fingerprint_id_helper_pqs8 = 12, + xnn_fingerprint_id_helper_qu8 = 13, + // Flag values + // + // Flag values are OR-ed so they need to avoid colliding. Not all + // operators use the same flags values. Flags values that don't overlap + // between operators may reuse the same value. + xnn_fingerprint_id_helper_example_flag = 1, + // The C preprocessor is obnoxious. For variadic arguments, there's no way to + // differentiate between an empty argument list and one argument. This value + // allows us to avoid bending around this issue when generating the + // fingerprint ID values. + // + // This value is OR-ed to the value by the COMBINE macro when no flag is + // passed. + xnn_fingerprint_id_helper_ = 0, +}; + +// Identifies a fingerprint. +// +// This should be retrievable from a fingerprint and allows identifying what +// that fingerprint identifies. +// +// Because fingerprints may be stored to be compared to past/future versions of +// XNNPack, **these values MUST NOT CHANGE**. +enum xnn_fingerprint_id { +#define XNN_FINGERPRINT_ID(operator, in, out, ...) \ + XNN_FINGERPRINT_ID_NAME(operator, in, out, __VA_ARGS__) = \ + XNN_FINGERPRINT_ID_VALUE(operator, in, out, __VA_ARGS__), +#include "fingerprint_id.h.inc" +#undef XNN_FINGERPRINT_ID + // The following values are aliases for special values in + // fingerprint_id.h.inc. + xnn_fingerprint_id_unknown = + xnn_fingerprint_id_unknown_unknown_unknown_unknown, + xnn_fingerprint_id_test = xnn_fingerprint_id_test_unknown_unknown_unknown, + xnn_fingerprint_id_no_fingerprint = + xnn_fingerprint_id_no_fingerprint_unknown_unknown_unknown, +}; + +enum xnn_fingerprint_id xnn_compute_fingerprint_id_value( + enum xnn_fingerprint_id_helper op, enum xnn_fingerprint_id_helper in, + enum xnn_fingerprint_id_helper out, enum xnn_fingerprint_id_helper weights, + ...); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // XNNPACK_SRC_OPERATORS_FINGERPRINT_ID_H_ diff --git a/src/operators/fingerprint_id.h.inc b/src/operators/fingerprint_id.h.inc new file mode 100644 index 00000000000..84d07b90f64 --- /dev/null +++ b/src/operators/fingerprint_id.h.inc @@ -0,0 +1,27 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +// This file has purposefully no include guard to be reused in places where we +// want to get the whole list of fingerprint IDs. +// +// This is the list of fingerprints identifiers. It can be included to get the +// list of fingerprints. +// +// `XNN_FINGERPRINT_ID` should be a macro that takes the following armuments: +// +// - operator: name of the operator. +// - in: type of the operator inputs (available types in fingerprint_id.h). +// - out: type of the operator inputs (available types in fingerprint_id.h). +// - weights: type of the operator inputs (available types in fingerprint_id.h). +// - flags...: optional list of extra qualifications. +// +// See fingerprint_id.h for an example. + +// LINT.IfChange(fingerprint_id) +XNN_FINGERPRINT_ID(unknown, unknown, unknown, unknown) +XNN_FINGERPRINT_ID(test, unknown, unknown, unknown) +XNN_FINGERPRINT_ID(test, f16, f32, qc8w, example_flag) +XNN_FINGERPRINT_ID(no_fingerprint, unknown, unknown, unknown) +// LINT.ThenChange(../xnnpack/fingerprint_check.c:fingerprint_compute) diff --git a/src/xnnpack/fingerprint_check.c b/src/xnnpack/fingerprint_check.c new file mode 100644 index 00000000000..3b4a1cefde4 --- /dev/null +++ b/src/xnnpack/fingerprint_check.c @@ -0,0 +1,33 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include "include/experimental.h" +#include "src/operators/fingerprint_id.h" + +static void compute_fingerprint(const enum xnn_fingerprint_id fingerprint_id) { + // LINT.IfChange(fingerprint_compute) + switch (fingerprint_id) { + case xnn_fingerprint_id_unknown: + case xnn_fingerprint_id_test: + case xnn_fingerprint_id_test_f16_f32_qc8w_example_flag: + case xnn_fingerprint_id_no_fingerprint: + return; + } + // LINT.ThenChange(../operators/fingerprint_id.h.inc:fingerprint_id) +} + +bool xnn_check_fingerprint(const struct xnn_fingerprint fingerprint) { + compute_fingerprint(fingerprint.id); + const struct xnn_fingerprint* reference_fingerprint = + xnn_get_fingerprint(fingerprint.id); + return reference_fingerprint && + reference_fingerprint->value == fingerprint.value; +} + diff --git a/test/BUILD.bazel b/test/BUILD.bazel index 494324782db..9bea49dff70 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -1020,6 +1020,17 @@ xnnpack_unit_test( ], ) +xnnpack_unit_test( + name = "fingerprint_cache_test", + srcs = ["fingerprint_cache.cc"], + deps = [ + "//:XNNPACK", + "//:cache", + "//:fingerprint_cache", + "//:fingerprint_id", + ], +) + xnnpack_unit_test( name = "weights_cache_test", srcs = ["weights-cache.cc"], diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cce34cbb3b7..cf83d6ecba9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -449,6 +449,14 @@ TARGET_LINK_LIBRARIES(buffer-test PRIVATE ADD_TEST(NAME buffer-test COMMAND buffer-test) SET_TARGET_PROPERTIES(buffer-test PROPERTIES CXX_EXTENSIONS YES) +ADD_EXECUTABLE(fingerprint-cache-test fingerprint_cache.cc) +TARGET_LINK_LIBRARIES(fingerprint-cache-test PRIVATE + xnnpack-test + xnnpack-fingerprint-cache + xnnpack-fingerprint-id) +ADD_TEST(NAME fingerprint-cache-test COMMAND fingerprint-cache-test) +SET_TARGET_PROPERTIES(fingerprint-cache-test PROPERTIES CXX_EXTENSIONS YES) + IF(XNNPACK_BUILD_LIBRARY) ADD_EXECUTABLE(weights-cache-test weights-cache.cc) TARGET_LINK_LIBRARIES(weights-cache-test PRIVATE XNNPACK pthreadpool GTest::gtest GTest::gtest_main) diff --git a/test/fingerprint_cache.cc b/test/fingerprint_cache.cc new file mode 100644 index 00000000000..391152cdbbf --- /dev/null +++ b/test/fingerprint_cache.cc @@ -0,0 +1,117 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "src/operators/fingerprint_cache.h" + +#include +#include +#include +#include +#include + +#include +#include +#include "include/experimental.h" +#include "include/xnnpack.h" +#include "src/operators/fingerprint_id.h" +#include "src/xnnpack/cache.h" + +using ::testing::Eq; +using ::testing::Not; +using ::testing::NotNull; +using ::testing::SizeIs; + +TEST(FingerPrintIdTest, AllIdsAreUnique) { +#define XNN_FINGERPRINT_ID(op, i, o, ...) \ + XNN_FINGERPRINT_ID_NAME(op, i, o, __VA_ARGS__), + std::vector ids{ +#include "src/operators/fingerprint_id.h.inc" + }; +#undef XNN_FINGERPRINT_ID + std::set unique_ids(ids.begin(), ids.end()); + EXPECT_THAT(unique_ids, SizeIs(ids.size())); +} + +TEST(FingerPrintIdTest, ComputeFingerprintIdValueWorks) { + EXPECT_THAT(xnn_compute_fingerprint_id_value( + xnn_fingerprint_id_helper_test, xnn_fingerprint_id_helper_f16, + xnn_fingerprint_id_helper_f32, xnn_fingerprint_id_helper_qc8w, + xnn_fingerprint_id_helper_example_flag, 0), + xnn_fingerprint_id_test_f16_f32_qc8w_example_flag); +} + +struct FingerprintCacheTest : testing::Test { + void SetUp() override { + xnn_initialize(nullptr); + xnn_clear_fingerprints(); + } + + void TearDown() override { + xnn_deinitialize(); + xnn_clear_fingerprints(); + } +}; + +TEST_F(FingerprintCacheTest, SetAndGetFingerprint) { + const struct xnn_fingerprint expected{/*id=*/xnn_fingerprint_id_test, + /*value=*/314}; + xnn_set_fingerprint(expected); + const struct xnn_fingerprint* fingerprint = xnn_get_fingerprint(expected.id); + ASSERT_THAT(fingerprint, NotNull()); + EXPECT_THAT(fingerprint->id, Eq(expected.id)); + EXPECT_THAT(fingerprint->value, Eq(expected.value)); +} + +TEST_F(FingerprintCacheTest, SetGetFingerprintMultipleTimesDoesntDeadlock) { + constexpr struct xnn_fingerprint finger1{/*id=*/xnn_fingerprint_id_test, + /*value=*/314}; + constexpr struct xnn_fingerprint finger2{/*id=*/xnn_fingerprint_id_test, + /*value=*/654}; + static_assert(finger1.id == finger2.id, + "This test checks that updating a fingerprint value doesn't " + "deadlock. Do use different ids for the two fingerprints."); + xnn_set_fingerprint(finger1); + xnn_set_fingerprint(finger1); + // Update the value of the fingerprint. + xnn_set_fingerprint(finger2); + xnn_set_fingerprint(finger2); + + xnn_get_fingerprint(finger1.id); + xnn_get_fingerprint(finger2.id); + xnn_get_fingerprint(finger1.id); + xnn_get_fingerprint(finger2.id); +} + +TEST_F(FingerprintCacheTest, InitializeAndFinalize) { + struct fingerprint_context context = + create_fingerprint_context(xnn_fingerprint_id_test); + EXPECT_THAT(context.status, Eq(xnn_status_uninitialized)); + EXPECT_THAT(context.fingerprint_id, Eq(xnn_fingerprint_id_test)); + finalize_fingerprint_context(&context); + EXPECT_THAT(xnn_get_fingerprint(xnn_fingerprint_id_test), NotNull()); +} + +TEST_F(FingerprintCacheTest, ReserveAndWrite) { + constexpr size_t kBufferSize = 8; + struct fingerprint_context context = + create_fingerprint_context(xnn_fingerprint_id_test); + uint8_t* buffer = reinterpret_cast( + context.cache.reserve_space(context.cache.context, kBufferSize)); + ASSERT_THAT(buffer, NotNull()); + std::iota(buffer, buffer + kBufferSize, 1); + // We just need a random key. + const xnn_weights_cache_look_up_key key = {0}; + context.cache.look_up_or_insert(context.cache.context, &key, buffer, + kBufferSize); + // The fingerprinting cache never returns a positive look up. + EXPECT_THAT(context.cache.look_up(context.cache.context, &key), + Eq(XNN_CACHE_NOT_FOUND)); + finalize_fingerprint_context(&context); + const xnn_fingerprint* fingerprint = + xnn_get_fingerprint(xnn_fingerprint_id_test); + ASSERT_THAT(fingerprint, NotNull()); + EXPECT_THAT(fingerprint->id, Eq(xnn_fingerprint_id_test)); + EXPECT_THAT(fingerprint->value, Not(Eq(0))); +}