From 69bd8a90e8c0dcca341199c9eb9deee8da286f92 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Tue, 29 Oct 2024 15:48:51 +0000 Subject: [PATCH 01/24] Squashed all JWT auth Add aspell Enable jwt-cpp in fasttest Add test + some minor improvements reduce unneeded possible clash points fix parsing create user identified with jwt refactor + fix not lowercase update test fix typo in docs fix logical_error some refactor fix alg in jwks fix jwks fix user auth method not being checked update docs better exception on no sub claim throw exception if algo not specified in jwk Support access token authorization of existing users Also possible to filter users by e-mail using regex fix token accessstorage Add Azure token processor, move JWKS logic to separate file remove docs that will be obsolete in future remove redundant resolve tokenCredentials on creation add basic docs for oauth add caching, cleanup bs fix credentials cast + some better code fix include fix invalid token handling add basic openid auth --- .../aspell-ignore/en/aspell-dict.txt | 7 + contrib/jwt-cpp-cmake/CMakeLists.txt | 7 +- .../external-authenticators/index.md | 4 +- .../operations/external-authenticators/jwt.md | 219 ++++++++++ .../external-authenticators/tokens.md | 108 +++++ src/Access/AccessControl.cpp | 18 + src/Access/AccessControl.h | 4 + src/Access/AccessTokenProcessor.cpp | 391 +++++++++++++++++ src/Access/AccessTokenProcessor.h | 129 ++++++ src/Access/Authentication.cpp | 13 +- src/Access/AuthenticationData.cpp | 21 +- src/Access/AuthenticationData.h | 4 + src/Access/Common/JWKSProvider.cpp | 89 ++++ src/Access/Common/JWKSProvider.h | 67 +++ src/Access/Credentials.cpp | 5 + src/Access/Credentials.h | 45 ++ src/Access/ExternalAuthenticators.cpp | 211 +++++++++- src/Access/ExternalAuthenticators.h | 24 ++ src/Access/IAccessStorage.cpp | 5 + src/Access/JWTValidator.cpp | 367 ++++++++++++++++ src/Access/JWTValidator.h | 69 +++ src/Access/TokenAccessStorage.cpp | 396 ++++++++++++++++++ src/Access/TokenAccessStorage.h | 79 ++++ src/Access/UsersConfigAccessStorage.cpp | 12 +- src/CMakeLists.txt | 1 + src/Parsers/Access/ASTAuthenticationData.cpp | 7 +- src/Parsers/Access/ASTCreateUserQuery.h | 4 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 15 + src/Parsers/Access/ParserCreateUserQuery.h | 4 +- src/Parsers/CommonParsers.h | 1 + src/Server/HTTP/authenticateUserByHTTP.cpp | 18 +- src/Server/TCPHandler.cpp | 18 + src/Server/TCPHandler.h | 1 + tests/integration/test_jwt_auth/__init__.py | 0 .../test_jwt_auth/configs/users.xml | 15 + .../test_jwt_auth/configs/validators.xml | 24 ++ .../helpers/generate_private_key.py | 21 + .../test_jwt_auth/helpers/jwt_jwk.py | 113 +++++ .../helpers/jwt_static_secret.py | 43 ++ .../test_jwt_auth/helpers/private_key_1 | 27 ++ .../test_jwt_auth/helpers/private_key_2 | 27 ++ .../test_jwt_auth/jwks_server/server.py | 33 ++ tests/integration/test_jwt_auth/test.py | 101 +++++ 43 files changed, 2746 insertions(+), 21 deletions(-) create mode 100644 docs/en/operations/external-authenticators/jwt.md create mode 100644 docs/en/operations/external-authenticators/tokens.md create mode 100644 src/Access/AccessTokenProcessor.cpp create mode 100644 src/Access/AccessTokenProcessor.h create mode 100644 src/Access/Common/JWKSProvider.cpp create mode 100644 src/Access/Common/JWKSProvider.h create mode 100644 src/Access/JWTValidator.cpp create mode 100644 src/Access/JWTValidator.h create mode 100644 src/Access/TokenAccessStorage.cpp create mode 100644 src/Access/TokenAccessStorage.h create mode 100644 tests/integration/test_jwt_auth/__init__.py create mode 100644 tests/integration/test_jwt_auth/configs/users.xml create mode 100644 tests/integration/test_jwt_auth/configs/validators.xml create mode 100644 tests/integration/test_jwt_auth/helpers/generate_private_key.py create mode 100644 tests/integration/test_jwt_auth/helpers/jwt_jwk.py create mode 100644 tests/integration/test_jwt_auth/helpers/jwt_static_secret.py create mode 100644 tests/integration/test_jwt_auth/helpers/private_key_1 create mode 100644 tests/integration/test_jwt_auth/helpers/private_key_2 create mode 100644 tests/integration/test_jwt_auth/jwks_server/server.py create mode 100644 tests/integration/test_jwt_auth/test.py diff --git a/ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt b/ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt index a94681f791ca..082663d5f1b9 100644 --- a/ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt +++ b/ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt @@ -286,6 +286,8 @@ DoubleDelta Doxygen Dresseler Durre +ECDSA +EdDSA ECMA ETag EachRow @@ -519,6 +521,8 @@ JoinAlgorithm JoinStrictness JumpConsistentHash Jupyter +jwks +JWKS KDevelop KafkaAssignedPartitions KafkaBackgroundReads @@ -3223,6 +3227,7 @@ uuid uuids uuidv vCPU +validators varPop varPopStable varSamp @@ -3240,6 +3245,8 @@ vectorscan vendoring verificationDepth verificationMode +verifier +verifiers versionedcollapsingmergetree vhost virtualized diff --git a/contrib/jwt-cpp-cmake/CMakeLists.txt b/contrib/jwt-cpp-cmake/CMakeLists.txt index 4cb8716bc68f..606c13d29de2 100644 --- a/contrib/jwt-cpp-cmake/CMakeLists.txt +++ b/contrib/jwt-cpp-cmake/CMakeLists.txt @@ -1,7 +1,4 @@ -set(ENABLE_JWT_CPP_DEFAULT OFF) -if(ENABLE_LIBRARIES AND CLICKHOUSE_CLOUD) - set(ENABLE_JWT_CPP_DEFAULT ON) -endif() +set(ENABLE_JWT_CPP_DEFAULT ON) option(ENABLE_JWT_CPP "Enable jwt-cpp library" ${ENABLE_JWT_CPP_DEFAULT}) @@ -20,4 +17,4 @@ set (JWT_CPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jwt-cpp/include") add_library (_jwt-cpp INTERFACE) target_include_directories(_jwt-cpp SYSTEM BEFORE INTERFACE ${JWT_CPP_INCLUDE_DIR}) -add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp) +add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp) \ No newline at end of file diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index 5a28003ad6c5..25d08d03fb7c 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -18,4 +18,6 @@ The following external authenticators and directories are supported: - [LDAP](/operations/external-authenticators/ldap#ldap-external-authenticator) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory) - Kerberos [Authenticator](/operations/external-authenticators/kerberos#kerberos-as-an-external-authenticator-for-existing-users) - [SSL X.509 authentication](/operations/external-authenticators/ssl-x509) -- HTTP [Authenticator](./http.md) \ No newline at end of file +- HTTP [Authenticator](./http.md) +- JWT [Authenticator](./jwt.md) +- Access Token [Authenticator](./tokens.md) diff --git a/docs/en/operations/external-authenticators/jwt.md b/docs/en/operations/external-authenticators/jwt.md new file mode 100644 index 000000000000..fbc36f8399f7 --- /dev/null +++ b/docs/en/operations/external-authenticators/jwt.md @@ -0,0 +1,219 @@ +--- +slug: /en/operations/external-authenticators/jwt +--- +# JWT +import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; + + + +Existing and properly configured ClickHouse users can be authenticated via JWT. + +Currently, JWT can only be used as an external authenticator for existing users, which are defined in `users.xml` or in local access control paths. +The username will be extracted from the JWT after validating the token expiration and against the signature. Signature can be validated by: +- static public key +- static JWKS +- received from the JWKS servers + +It is mandatory for a JWT tot indicate the name of the ClickHouse user under `"sub"` claim, otherwise it will not be accepted. + +A JWT may additionally be verified by checking the JWT payload. +In this case, the occurrence of specified claims from the user settings in the JWT payload is checked. +See [Enabling JWT authentication in `users.xml`](#enabling-jwt-auth-in-users-xml) + +To use JWT authentication, JWT validators must be configured in ClickHouse config. + + +## Enabling JWT validators in ClickHouse {#enabling-jwt-validators-in-clickhouse} + +To enable JWT validators, add `token_validators` section in `config.xml`. This section may contain several JWT verifiers, minimum is 1. + +### Verifying JWT signature using static key {$verifying-jwt-signature-using-static-key} + +**Example** +```xml + + + + + HS256 + my_static_secret + + + +``` + +#### Parameters: + +- `algo` - Algorithm for validate signature. Supported: + + | HMAC | RSA | ECDSA | PSS | EdDSA | + |-------| ----- | ------ | ----- | ------- | + | HS256 | RS256 | ES256 | PS256 | Ed25519 | + | HS384 | RS384 | ES384 | PS384 | Ed448 | + | HS512 | RS512 | ES512 | PS512 | | + | | | ES256K | | | + Also support None. +- `static_key` - key for symmetric algorithms. Mandatory for `HS*` family algorithms. +- `static_key_in_base64` - indicates if the `static_key` key is base64-encoded. Optional, default: `False`. +- `public_key` - public key for asymmetric algorithms. Mandatory except for `HS*` family algorithms and `None`. +- `private_key` - private key for asymmetric algorithms. Optional. +- `public_key_password` - public key password. Optional. +- `private_key_password` - private key password. Optional. + +### Verifying JWT signature using static JWKS {$verifying-jwt-signature-using-static-jwks} + +:::note +Only RS* family algorithms are supported! +::: + +**Example** +```xml + + + + + {"keys": [{"kty": "RSA", "alg": "RS256", "kid": "mykid", "n": "_public_key_mod_", "e": "AQAB"}]} + + + +``` + +#### Parameters: +- `static_jwks` - content of JWKS in json +- `static_jwks_file` - path to file with JWKS + +:::note +Only one of `static_jwks` or `static_jwks_file` keys must be present in one verifier +::: + +### Verifying JWT signature using JWKS servers {$verifying-jwt-signature-using-static-jwks} + +**Example** +```xml + + + + + http://localhost:8000/.well-known/jwks.json + 1000 + 1000 + 1000 + 3 + 50 + 1000 + 300000 + + + +``` + +#### Parameters: + +- `uri` - JWKS endpoint. Mandatory. +- `refresh_ms` - Period for resend request for refreshing JWKS. Optional, default: 300000. + +Timeouts in milliseconds on the socket used for communicating with the server (optional): +- `connection_timeout_ms` - Default: 1000. +- `receive_timeout_ms` - Default: 1000. +- `send_timeout_ms` - Default: 1000. + +Retry parameters (optional): +- `max_tries` - The maximum number of attempts to make an authentication request. Default: 3. +- `retry_initial_backoff_ms` - The backoff initial interval on retry. Default: 50. +- `retry_max_backoff_ms` - The maximum backoff interval. Default: 1000. + +### Verifying access tokens {$verifying-access-tokens} + +Access tokens that are not JWT (and thus no data can be extracted from the token directly) need to be resolved by external providers. + +**Example** +```xml + + + + + google + + + +``` + +#### Parameters: + +- `provider` - name of provider that will be used for token processing. Mandatory parameter. Possible options: `google`. + + +### Enabling JWT authentication in `users.xml` {#enabling-jwt-auth-in-users-xml} + +In order to enable JWT authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. + +Parameters: +- `claims` - An optional string containing a json object that should be contained in the token payload. + +Example (goes into `users.xml`): +```xml + + + + + + {"resource_access":{"account": {"roles": ["view-profile"]}}} + + + +``` + +Here, the JWT payload must contain `["view-profile"]` on path `resource_access.account.roles`, otherwise authentication will not succeed even with a valid JWT. + +``` +{ +... + "resource_access": { + "account": { + "roles": ["view-profile"] + } + }, +... +} +``` + +:::note +JWT authentication cannot be used together with any other authentication method. The presence of any other sections like `password` alongside `jwt` will force ClickHouse to shut down. +::: + +### Enabling JWT authentication using SQL {#enabling-jwt-auth-using-sql} + +When [SQL-driven Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled in ClickHouse, users identified by JWT authentication can also be created using SQL statements. + +```sql +CREATE USER my_user IDENTIFIED WITH jwt CLAIMS '{"resource_access":{"account": {"roles": ["view-profile"]}}}' +``` + +Or without additional JWT payload checks: + +```sql +CREATE USER my_user IDENTIFIED WITH jwt +``` + +## JWT authentication examples {#jwt-authentication-examples} + +#### Console client + +``` +clickhouse-client -jwt +``` + +#### HTTP requests + +``` +curl 'http://localhost:8080/?' \ + -H 'Authorization: Bearer ' \ + -H 'Content type: text/plain;charset=UTF-8' \ + --data-raw 'SELECT current_user()' +``` +:::note +ClickHouse will look for a JWT token in (by priority): +1. `X-ClickHouse-JWT-Token` header. +2. `Authorization` header. +3. `token` request parameter. In this case, the "Bearer" prefix should not exist. +::: diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md new file mode 100644 index 000000000000..6340e318237e --- /dev/null +++ b/docs/en/operations/external-authenticators/tokens.md @@ -0,0 +1,108 @@ +--- +slug: /en/operations/external-authenticators/oauth +title: "OAuth 2.0" +--- +import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; + + + +OAuth 2.0 access tokens can be used to authenticate ClickHouse users. This works in two ways: + +- Existing users (defined in `users.xml` or in local access control paths) can be authenticated with access token if this user can be `IDENTIFIED WITH jwt`. +- Use Identity Provider (IdP) as an external user directory and allow locally undefined users to be authenticated with a token if it is valid and recognized by the provider. + +Though this authentication method is different from JWT authentication, it works under the same authentication method to maintain better compatibility. + +For both of these approaches a definition of `access_token_processors` is mandatory. + +## Access Token Processors + +To define an access token processor, add `access_token_processors` section to `config.xml`. Example: +```xml + + + + Google + ^[A-Za-z0-9._%+-]+@example\.com$ + 600 + + + azure + CLIENT_ID + TENANT_ID + + + +``` + +:::note +Different providers have different sets of parameters. +::: + +**Parameters** + +- `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure". +- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. +- `email_filter` -- Regex for validation of user emails. Optional parameter, only for Google IdP. +- `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, only for Azure IdP. +- `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, only for Azure IdP. + +### Tokens cache +To reduce number of requests to IdP, tokens are cached internally for no longer then `cache_lifetime` seconds. +If token expires sooner than `cache_lifetime`, then cache entry for this token will only be valid while token is valid. +If token lifetime is longer than `cache_lifetime`, cache entry for this token will be valid for `cache_lifetime`. + +## IdP as External Authenticator {#idp-external-authenticator} + +Locally defined users can be authenticated with an access token. To allow this, `jwt` must be specified as user's authentication method. Example: + +```xml + + + + + + + + + + +``` + +At each login attempt, ClickHouse will attempt to validate token and get user info against every defined access token provider. + +When SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled, users that are authenticated with tokens can also be created using the [CREATE USER](/docs/en/sql-reference/statements/create/user.md#create-user-statement) statement. + +Query: + +```sql +CREATE USER my_user IDENTIFIED WITH jwt; +``` + +## Identity Provider as an External User Directory {#idp-external-user-directory} + +If there is no suitable user pre-defined in ClickHouse, authentication is still possible: Identity Provider can be used as source of user information. +To allow this, add `token` section to the `users_directories` section of the `config.xml` file. + +At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. +If the user is not defined, ClickHouse will treat user as externally defined, and will try to validate the token and get user information from the specified processor. +If validated successfully, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. +All this implies that the SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled and roles are created using the [CREATE ROLE](/docs/en/sql-reference/statements/create/role.md#create-role-statement) statement. + +**Example** + +```xml + + + gogoogle + + + + + +``` + +**Parameters** + +- `server` — Name of one of processors defined in `access_token_processors` config section described above. This parameter is mandatory and cannot be empty. +- `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the IdP. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 71dfdf41d153..0cd64717b60a 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -421,6 +422,12 @@ void AccessControl::addLDAPStorage(const String & storage_name_, const Poco::Uti LOG_DEBUG(getLogger(), "Added {} access storage '{}', LDAP server name: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getLDAPServerName()); } +void AccessControl::addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) +{ + auto new_storage = std::make_shared(storage_name_, *this, config_, prefix_); + addStorage(new_storage); + LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName()); +} void AccessControl::addStoragesFromUserDirectoriesConfig( const Poco::Util::AbstractConfiguration & config, @@ -446,6 +453,8 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( type = DiskAccessStorage::STORAGE_TYPE; else if (type == "ldap") type = LDAPAccessStorage::STORAGE_TYPE; + else if (type == "token") + type = TokenAccessStorage::STORAGE_TYPE; String name = config.getString(prefix + ".name", type); @@ -479,6 +488,10 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( bool allow_backup = config.getBool(prefix + ".allow_backup", true); addReplicatedStorage(name, zookeeper_path, get_zookeeper_function, allow_backup); } + else if (type == TokenAccessStorage::STORAGE_TYPE) + { + addTokenStorage(name, config, prefix); + } else throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown storage type '{}' at {} in config", type, prefix); } @@ -708,6 +721,11 @@ bool AccessControl::isNoPasswordAllowed() const return allow_no_password; } +bool AccessControl::isJWTEnabled() const +{ + return external_authenticators->isJWTAllowed(); +} + void AccessControl::setPlaintextPasswordAllowed(bool allow_plaintext_password_) { allow_plaintext_password = allow_plaintext_password_; diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index b230fd81b1c7..76c0d63c216f 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -93,6 +93,8 @@ class AccessControl : public MultipleAccessStorage /// Adds LDAPAccessStorage which allows querying remote LDAP server for user info. void addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_); + void addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_); + void addReplicatedStorage(const String & storage_name, const String & zookeeper_path, const zkutil::GetZooKeeper & get_zookeeper_function, @@ -155,6 +157,8 @@ class AccessControl : public MultipleAccessStorage void setNoPasswordAllowed(bool allow_no_password_); bool isNoPasswordAllowed() const; + bool isJWTEnabled() const; + /// Allows users with plaintext password (by default it's allowed). void setPlaintextPasswordAllowed(bool allow_plaintext_password_); bool isPlaintextPasswordAllowed() const; diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp new file mode 100644 index 000000000000..9d6cc907afe6 --- /dev/null +++ b/src/Access/AccessTokenProcessor.cpp @@ -0,0 +1,391 @@ +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + /// The JSON reply from provider has only a few key-value pairs, so no need for any advanced parsing. + /// Reduce complexity by using picojson. + picojson::object parseJSON(const String & json_string) { + picojson::value jsonValue; + std::string err = picojson::parse(jsonValue, json_string); + + if (!err.empty()) { + throw std::runtime_error("JSON parsing error: " + err); + } + + if (!jsonValue.is()) { + throw std::runtime_error("JSON is not an object"); + } + + return jsonValue.get(); + } + + template + ValueType getValueByKey(const picojson::object & jsonObject, const std::string & key) { + auto it = jsonObject.find(key); // Find the key in the object + if (it == jsonObject.end()) + { + throw std::runtime_error("Key not found: " + key); + } + + const picojson::value & value = it->second; + if (!value.is()) { + throw std::runtime_error("Value for key '" + key + "' has incorrect type."); + } + + return value.get(); + } + + picojson::object getObjectFromURI(const Poco::URI & uri, const String & token = "") + { + Poco::Net::HTTPResponse response; + std::ostringstream responseString; + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, uri.getPathAndQuery()}; + if (!token.empty()) + request.add("Authorization", "Bearer " + token); + + if (uri.getScheme() == "https") { + Poco::Net::HTTPSClientSession session(uri.getHost(), uri.getPort()); + session.sendRequest(request); + Poco::StreamCopier::copyStream(session.receiveResponse(response), responseString); + } + else + { + Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort()); + session.sendRequest(request); + Poco::StreamCopier::copyStream(session.receiveResponse(response), responseString); + } + + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, + "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), + response.getReason()); + + try + { + return parseJSON(responseString.str()); + } + catch (const std::runtime_error & e) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse server response: {}", e.what()); + } + } +} + + +[[maybe_unused]] const Poco::URI GoogleAccessTokenProcessor::token_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/tokeninfo"); +const Poco::URI GoogleAccessTokenProcessor::user_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/userinfo"); + +const Poco::URI AzureAccessTokenProcessor::user_info_uri = Poco::URI("https://graph.microsoft.com/oidc/userinfo"); + + +std::unique_ptr IAccessTokenProcessor::parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name) +{ + if (config.hasProperty(prefix + ".provider")) + { + String provider = Poco::toLower(config.getString(prefix + ".provider")); + + String email_regex_str = config.hasProperty(prefix + ".email_filter") ? config.getString( + prefix + ".email_filter") : ""; + + UInt64 cache_lifetime = config.hasProperty(prefix + ".cache_lifetime") ? config.getUInt64( + prefix + ".cache_lifetime") : 3600; + + if (provider == "google") + { + return std::make_unique(name, cache_lifetime, email_regex_str); + } + else if (provider == "azure") + { + if (!config.hasProperty(prefix + ".client_id")) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: client_id must be specified", name); + + if (!config.hasProperty(prefix + ".tenant_id")) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: tenant_id must be specified", name); + + String tenant_id_str = config.getString(prefix + ".tenant_id"); + + return std::make_unique(name, cache_lifetime, email_regex_str, tenant_id_str); + } + else if (provider == "openid") + { + bool is_auto = config.hasProperty(prefix + ".configuration_endpoint"); + bool is_manual = config.hasProperty(prefix + ".userinfo_endpoint") && + config.hasProperty(prefix + ".token_introspection_endpoint") && + (config.hasProperty(prefix + ".userinfo_endpoint") == config.hasProperty(prefix + ".token_introspection_endpoint")); + + if (is_auto && !is_manual) + { + return std::make_unique(name, cache_lifetime, email_regex_str, config.getString(prefix + ".configuration_endpoint")); + } + else if (!is_auto && is_manual) + { + return std::make_unique(name, cache_lifetime, email_regex_str, config.getString(prefix + ".userinfo_endpoint"), config.getString(prefix + ".token_introspection_endpoint")); + } + + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse access token processor {}: " + "Either configuration_endpoint or both userinfo_endpoint and token_introspection_endpoint shall be specified", name); + } + else + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: unknown provider type {}", name, provider); + } + + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: provider name must be specified", name); +} + + +bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +{ + const String & token = credentials.getToken(); + + auto user_info = getUserInfo(token); + String user_name = user_info["sub"]; + bool has_email = user_info.contains("email"); + + if (email_regex.ok()) + { + if (!has_email) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to validate {} by e-mail", name, user_name); + return false; + } + + /// Additionally validate user email to match regex from config. + if (!RE2::FullMatch(user_info["email"], email_regex)) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to authenticate user {}: e-mail address is not permitted.", name, user_name); + return false; + } + + } + + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(user_name); + + auto token_info = getObjectFromURI(Poco::URI(token_info_uri), token); + if (token_info.contains("exp")) + const_cast(credentials).setExpiresAt(std::chrono::system_clock::from_time_t((getValueByKey(token_info, "exp")))); + + /// Groups info can only be retrieved if user email is known. + /// If no email found in user info, we skip this step and there are no external groups for the user. + if (has_email) + { + std::set external_groups_names; + const Poco::URI get_groups_uri = Poco::URI("https://cloudidentity.googleapis.com/v1/groups/-/memberships:searchDirectGroups?query=member_key_id==" + user_info["email"] + "'"); + + try + { + auto groups_response = getObjectFromURI(get_groups_uri, token); + + if (!groups_response.contains("memberships") || !groups_response["memberships"].is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups: invalid content in response from server", name); + return true; + } + + for (const auto & group: groups_response["memberships"].get()) + { + if (!group.is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups: invalid content in response from server", name); + continue; + } + + auto group_data = group.get(); + String group_name = getValueByKey(group_data["groupKey"].get(), "id"); + external_groups_names.insert(group_name); + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: User {}: new external group {}", name, user_name, group_name); + } + + const_cast(credentials).setGroups(external_groups_names); + } + catch (const Exception & e) + { + /// Could not get groups info. Log it and skip it. + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups, no external roles will be mapped. reason: {}", name, e.what()); + return true; + } + } + + return true; +} + +std::unordered_map GoogleAccessTokenProcessor::getUserInfo(const String & token) const +{ + std::unordered_map user_info_map; + picojson::object user_info_json = getObjectFromURI(user_info_uri, token); + + try + { + user_info_map["email"] = getValueByKey(user_info_json, "email"); + user_info_map["sub"] = getValueByKey(user_info_json, "sub"); + return user_info_map; + } + catch (std::runtime_error & e) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Failed to get user info with token: {}", name, e.what()); + } +} + + +bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +{ + /// Token is a JWT in this case, but we cannot directly verify it against Azure AD JWKS. + /// We will not trust user data in this token except for 'exp' value to determine caching duration. + /// Explanation here: https://stackoverflow.com/questions/60778634/failing-signature-validation-of-jwt-tokens-from-azure-ad + /// Let Azure validate it: only valid tokens will be accepted. + /// Use GET https://graph.microsoft.com/oidc/userinfo to verify token and get sub at the same time + + const String & token = credentials.getToken(); + + try + { + String username = validateTokenAndGetUsername(token); + if (!username.empty()) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + } + else + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username with token", name); + + } + catch (...) + { + return false; + } + + try + { + const_cast(credentials).setExpiresAt(jwt::decode(token).get_expires_at()); + } + catch (...) { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: No expiration data found in a valid token, will use default cache lifetime", name); + } + + std::set external_groups_names; + const Poco::URI get_groups_uri = Poco::URI("https://graph.microsoft.com/v1.0/me/memberOf"); + + try + { + auto groups_response = getObjectFromURI(get_groups_uri, token); + + if (!groups_response.contains("value") || !groups_response["value"].is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups: invalid content in response from server", name); + return true; + } + + picojson::array groups_array = groups_response["value"].get(); + + for (const auto & group: groups_array) + { + /// Got some invalid response. Ignore this, log this. + if (!group.is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups: invalid content in response from server", name); + continue; + } + + auto group_data = group.get(); + String group_name = getValueByKey(group_data, "id"); + external_groups_names.insert(group_name); + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: User {}: new external group {}", name, credentials.getUserName(), group_name); + } + } + catch (const Exception & e) + { + /// Could not get groups info. Log it and skip it. + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups, no external roles will be mapped. reason: {}", name, e.what()); + return true; + } + + const_cast(credentials).setGroups(external_groups_names); + + return true; +} + +String AzureAccessTokenProcessor::validateTokenAndGetUsername(const String & token) const +{ + picojson::object user_info_json = getObjectFromURI(user_info_uri, token); + return getValueByKey(user_info_json, "sub"); +} + +OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str, + const String & openid_config_endpoint_) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) +{ + const picojson::object openid_config = getObjectFromURI(Poco::URI(openid_config_endpoint_)); + + if (!openid_config.contains("userinfo_endpoint") || !openid_config.contains("introspection_endpoint")) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Cannot extract userinfo_endpoint or introspection_endpoint from OIDC configuration, consider manual configuration.", name); +} + +bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +{ + const String & token = credentials.getToken(); + + try + { + String username = validateTokenAndGetUsername(token); + if (!username.empty()) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + } + else + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username with token", name); + + } + catch (...) + { + return false; + } + + return true; + + /// TODO: add proper groups functionality +// try +// { +// const_cast(credentials).setExpiresAt(jwt::decode(token).get_expires_at()); +// } +// catch (...) { +// LOG_TRACE(getLogger("AccessTokenProcessor"), +// "{}: No expiration data found in a valid token, will use default cache lifetime", name); +// } +} + +String OpenIDAccessTokenProcessor::validateTokenAndGetUsername(const String & token) const +{ + picojson::object user_info_json = getObjectFromURI(userinfo_endpoint, token); + return getValueByKey(user_info_json, "sub"); +} + +} diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h new file mode 100644 index 000000000000..125b6d87b5c9 --- /dev/null +++ b/src/Access/AccessTokenProcessor.h @@ -0,0 +1,129 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +//class GoogleAccessTokenProcessor; + +class IAccessTokenProcessor +{ +public: + IAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str) + : name(name_), + cache_invalidation_interval(cache_invalidation_interval_), + email_regex(email_regex_str) + { + if (!email_regex_str.empty()) + { + /// Later, we will use .ok() to determine whether there was a regex specified in config or not. + if (!email_regex.ok()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Invalid regex in definition of access token processor {}", name); + } + } + + virtual ~IAccessTokenProcessor() = default; + + String getName() { return name; } + UInt64 getCacheInvalidationInterval() { return cache_invalidation_interval; } + + virtual bool resolveAndValidate(const TokenCredentials & credentials) = 0; + + static std::unique_ptr parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name); + +protected: + const String name; + const UInt64 cache_invalidation_interval; + re2::RE2 email_regex; + + bool valid; +}; + + +class GoogleAccessTokenProcessor : public IAccessTokenProcessor +{ +public: + GoogleAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) {} + + bool resolveAndValidate(const TokenCredentials & credentials) override; + +private: + [[maybe_unused]] static const Poco::URI token_info_uri; + static const Poco::URI user_info_uri; + + std::unordered_map getUserInfo(const String & token) const; +}; + + +class AzureAccessTokenProcessor : public IAccessTokenProcessor +{ +public: + AzureAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str, + const String & tenant_id_) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), + jwks_uri_str("https://login.microsoftonline.com/" + tenant_id_ + "/discovery/v2.0/keys") {} + + bool resolveAndValidate(const TokenCredentials & credentials) override; +private: + static const Poco::URI user_info_uri; + + const String jwks_uri_str; + + String validateTokenAndGetUsername(const String & token) const; +}; + +class OpenIDAccessTokenProcessor : public IAccessTokenProcessor +{ +public: + /// Obtain endpoints from openid-configuration URL + OpenIDAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str, + const String & openid_config_endpoint_); + + /// Specify endpoints manually + OpenIDAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str, + const String & userinfo_endpoint_, + const String & token_introspection_endpoint_) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), + userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_) {} + + bool resolveAndValidate(const TokenCredentials & credentials) override; +private: + const Poco::URI userinfo_endpoint; + const Poco::URI token_introspection_endpoint; + + + String validateTokenAndGetUsername(const String & token) const; +}; + +} diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 03e47ed3b6d6..a7ae9022ec6b 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -298,7 +298,7 @@ bool Authentication::areCredentialsValid( const ClientInfo & client_info, SettingsChanges & settings) { - if (!credentials.isReady()) + if (!typeid_cast(&credentials) && !credentials.isReady()) return false; if (const auto * gss_acceptor_context = typeid_cast(&credentials)) @@ -340,6 +340,17 @@ bool Authentication::areCredentialsValid( } #endif + if (const auto * token_credentials = typeid_cast(&credentials)) + { + if (authentication_method.getType() != AuthenticationType::JWT) + return false; + + if (external_authenticators.checkJWTClaims(authentication_method.getJWTClaims(), *token_credentials)) + return true; + + return external_authenticators.checkAccessTokenCredentials(*token_credentials); + } + if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index 7492ce710f35..ca3fa4426564 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -14,6 +14,8 @@ #include #include +#include + #include "config.h" #if USE_SSL @@ -373,7 +375,10 @@ std::shared_ptr AuthenticationData::toAST() const } case AuthenticationType::JWT: { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); + const auto & claims = getJWTClaims(); + if (!claims.empty()) + node->children.push_back(std::make_shared(claims)); + break; } case AuthenticationType::KERBEROS: { @@ -643,6 +648,20 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que auth_data.setHTTPAuthenticationServerName(server); auth_data.setHTTPAuthenticationScheme(scheme); } + else if (query.type == AuthenticationType::JWT) + { + if (!args.empty()) + { + String value = checkAndGetLiteralArgument(args[0], "claims"); + picojson::value json_obj; + auto error = picojson::parse(json_obj, value); + if (!error.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: {}", error); + if (!json_obj.is()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: is not an object"); + auth_data.setJWTClaims(value); + } + } else { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected ASTAuthenticationData structure"); diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h index 187a17dca948..6bfeae4cea3b 100644 --- a/src/Access/AuthenticationData.h +++ b/src/Access/AuthenticationData.h @@ -79,6 +79,9 @@ class AuthenticationData time_t getValidUntil() const { return valid_until; } void setValidUntil(time_t valid_until_) { valid_until = valid_until_; } + const String & getJWTClaims() const { return jwt_claims; } + void setJWTClaims(const String & jwt_claims_) { jwt_claims = jwt_claims_; } + friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs); friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); } @@ -117,6 +120,7 @@ class AuthenticationData String http_auth_server_name; HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC; time_t valid_until = 0; + String jwt_claims; }; } diff --git a/src/Access/Common/JWKSProvider.cpp b/src/Access/Common/JWKSProvider.cpp new file mode 100644 index 000000000000..94ee5e04cafe --- /dev/null +++ b/src/Access/Common/JWKSProvider.cpp @@ -0,0 +1,89 @@ +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +jwt::jwks JWKSClient::getJWKS() +{ + std::shared_lock lock(mutex); + + auto now = std::chrono::high_resolution_clock::now(); + auto diff = std::chrono::duration(now - last_request_send).count(); + + if (diff < refresh_ms) { + jwt::jwks result(cached_jwks); + return result; + } + + Poco::Net::HTTPResponse response; + std::ostringstream responseString; + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, jwks_uri.getPathAndQuery()}; + + if (jwks_uri.getScheme() == "https") { + Poco::Net::HTTPSClientSession session = Poco::Net::HTTPSClientSession(jwks_uri.getHost(), jwks_uri.getPort()); + session.sendRequest(request); + std::istream & responseStream = session.receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !responseStream) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + Poco::StreamCopier::copyStream(responseStream, responseString); + } else { + Poco::Net::HTTPClientSession session = Poco::Net::HTTPClientSession(jwks_uri.getHost(), jwks_uri.getPort()); + session.sendRequest(request); + std::istream & responseStream = session.receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !responseStream) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + Poco::StreamCopier::copyStream(responseStream, responseString); + } + + last_request_send = std::chrono::high_resolution_clock::now(); + + jwt::jwks parsed_jwks; + + try { + parsed_jwks = jwt::parse_jwks(responseString.str()); + } + catch (const Exception & e) { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse JWKS: {}", e.what()); + } + + cached_jwks = std::move(parsed_jwks); + return cached_jwks; +} + +StaticJWKSParams::StaticJWKSParams(const std::string &static_jwks_, const std::string &static_jwks_file_) +{ + if (static_jwks_.empty() && static_jwks_file_.empty()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "JWT validator misconfigured: `static_jwks` or `static_jwks_file` keys must be present in static JWKS validator configuration"); + if (!static_jwks_.empty() && !static_jwks_file_.empty()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "JWT validator misconfigured: `static_jwks` and `static_jwks_file` keys cannot both be present in static JWKS validator configuration"); + + static_jwks = static_jwks_; + static_jwks_file = static_jwks_file_; +} + +StaticJWKS::StaticJWKS(const StaticJWKSParams ¶ms) +{ + String content = String(params.static_jwks); + if (!params.static_jwks_file.empty()) { + std::ifstream ifs(params.static_jwks_file); + content = String((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); + } + auto keys = jwt::parse_jwks(content); + jwks = std::move(keys); +} + +} diff --git a/src/Access/Common/JWKSProvider.h b/src/Access/Common/JWKSProvider.h new file mode 100644 index 000000000000..773208a138aa --- /dev/null +++ b/src/Access/Common/JWKSProvider.h @@ -0,0 +1,67 @@ +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +class IJWKSProvider +{ +public: + virtual ~IJWKSProvider() = default; + + virtual jwt::jwks getJWKS() = 0; +}; + +class JWKSClient : public IJWKSProvider +{ +public: + explicit JWKSClient(const String & uri, const size_t refresh_ms_): refresh_ms(refresh_ms_), jwks_uri(uri) {} + + ~JWKSClient() override = default; + JWKSClient(const JWKSClient &) = delete; + JWKSClient(JWKSClient &&) = delete; + JWKSClient &operator=(const JWKSClient &) = delete; + JWKSClient &operator=(JWKSClient &&) = delete; + + jwt::jwks getJWKS() override; + +private: + size_t refresh_ms; + Poco::URI jwks_uri; + + std::shared_mutex mutex; + jwt::jwks cached_jwks; + std::chrono::time_point last_request_send; +}; + +struct StaticJWKSParams +{ + StaticJWKSParams(const std::string &static_jwks_, const std::string &static_jwks_file_); + + String static_jwks; + String static_jwks_file; +}; + +class StaticJWKS : public IJWKSProvider +{ +public: + explicit StaticJWKS(const StaticJWKSParams ¶ms); + +private: + jwt::jwks getJWKS() override + { + return jwks; + } + + jwt::jwks jwks; +}; + +} diff --git a/src/Access/Credentials.cpp b/src/Access/Credentials.cpp index 4887d0545656..c60fb3cfea67 100644 --- a/src/Access/Credentials.cpp +++ b/src/Access/Credentials.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -9,6 +10,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int AUTHENTICATION_FAILED; } Credentials::Credentials(const String & user_name_) @@ -100,4 +102,7 @@ const String & BasicCredentials::getPassword() const return password; } +/// Unless the token is validated, we will not use any data from it, including username. +TokenCredentials::TokenCredentials(const String & token_) : Credentials(""), token(token_), expires_at(std::chrono::system_clock::now() + std::chrono::hours(1)) {} + } diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index f98eb31ff0a2..bb81ef93a15c 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -16,6 +16,8 @@ namespace Poco::Net class SocketAddress; } +#include + namespace DB { @@ -195,4 +197,47 @@ class SSHPTYCredentials : public Credentials #endif +class TokenCredentials : public Credentials +{ +public: + explicit TokenCredentials(const String & token_); + + const String & getToken() const + { + if (token.empty()) + { + throwNotReady(); + } + return token; + } + void setUserName(const String & user_name_) + { + user_name = user_name_; + if (!user_name.empty()) + { + is_ready = true; + } + } + std::set getGroups() const + { + return groups; + } + void setGroups(const std::set & groups_) + { + groups = groups_; + } + std::optional getExpiresAt() const + { + return expires_at; + } + void setExpiresAt(std::chrono::system_clock::time_point expires_at_) + { + expires_at = expires_at_; + } +private: + String token; + std::set groups; + std::optional expires_at; +}; + } diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index ca61b55dc5dc..00ec7a7a5f5c 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -1,7 +1,12 @@ +#include +#include +#include #include #include #include #include +#include "Common/Logger.h" +#include "Common/logger_useful.h" #include #include #include @@ -12,6 +17,8 @@ #include #include +#include +#include #include #include @@ -263,7 +270,6 @@ HTTPAuthClientParams parseHTTPAuthParams(const Poco::Util::AbstractConfiguration return http_auth_params; } - } void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix) @@ -281,6 +287,13 @@ void ExternalAuthenticators::resetImpl() ldap_client_params_blueprint.clear(); ldap_caches.clear(); kerberos_params.reset(); + jwt_validators.clear(); +} + +bool ExternalAuthenticators::isJWTAllowed() const +{ + std::lock_guard lock(mutex); + return !jwt_validators.empty(); } void ExternalAuthenticators::reset() @@ -289,6 +302,52 @@ void ExternalAuthenticators::reset() resetImpl(); } +void parseJWTValidators(std::unordered_map> & jwt_validators, + const Poco::Util::AbstractConfiguration & config, + const String & jwt_validators_config, + LoggerPtr log) +{ + Poco::Util::AbstractConfiguration::Keys jwt_validators_keys; + config.keys(jwt_validators_config, jwt_validators_keys); + jwt_validators.clear(); + for (const auto & jwt_validator : jwt_validators_keys) + { + if (jwt_validator == "settings_key") continue; + String prefix = fmt::format("{}.{}", jwt_validators_config, jwt_validator); + try + { + jwt_validators[jwt_validator] = IJWTValidator::parseJWTValidator(config, prefix, jwt_validator); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse JWT validator" + backQuote(jwt_validator)); + } + } +} + +void parseAccessTokenProcessors(std::unordered_map> & access_token_processors, + const Poco::Util::AbstractConfiguration & config, + const String & access_token_processors_config, + LoggerPtr log) +{ + Poco::Util::AbstractConfiguration::Keys access_token_processors_keys; + config.keys(access_token_processors_config, access_token_processors_keys); + access_token_processors.clear(); + + for (const auto & processor : access_token_processors_keys) + { + String prefix = fmt::format("{}.{}", access_token_processors_config, processor); + try + { + access_token_processors[processor] = IAccessTokenProcessor::parseTokenProcessor(config, prefix, processor); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse access token processor" + backQuote(processor)); + } + } +} + void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, LoggerPtr log) { std::lock_guard lock(mutex); @@ -300,8 +359,12 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur std::size_t ldap_servers_key_count = 0; std::size_t kerberos_keys_count = 0; std::size_t http_auth_server_keys_count = 0; + std::size_t jwt_validators_count = 0; + std::size_t access_token_processors_count = 0; const String http_auth_servers_config = "http_authentication_servers"; + const String jwt_validators_config = "jwt_validators"; + const String access_token_processors_config = "access_token_processors"; for (auto key : all_keys) { @@ -314,6 +377,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur ldap_servers_key_count += (key == "ldap_servers"); kerberos_keys_count += (key == "kerberos"); http_auth_server_keys_count += (key == http_auth_servers_config); + jwt_validators_count += (key == jwt_validators_config); + access_token_processors_count += (key == access_token_processors_config); } if (ldap_servers_key_count > 1) @@ -325,6 +390,12 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur if (http_auth_server_keys_count > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple http_authentication_servers sections are not allowed"); + if (jwt_validators_count > 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", jwt_validators_config); + + if (access_token_processors_count > 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", access_token_processors_config); + Poco::Util::AbstractConfiguration::Keys http_auth_server_names; config.keys(http_auth_servers_config, http_auth_server_names); http_auth_servers.clear(); @@ -379,6 +450,9 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur { tryLogCurrentException(log, "Could not parse Kerberos section"); } + + parseJWTValidators(jwt_validators, config, jwt_validators_config, log); + parseAccessTokenProcessors(access_token_processors, config, access_token_processors_config, log); } static UInt128 computeParamsHash(const LDAPClient::Params & params, const LDAPClient::RoleSearchParamsList * role_search_params) @@ -547,7 +621,7 @@ GSSAcceptorContext::Params ExternalAuthenticators::getKerberosParams() const return kerberos_params.value(); } -HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const String& server) const +HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const String & server) const { std::lock_guard lock{mutex}; @@ -557,6 +631,139 @@ HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const S return it->second; } +bool ExternalAuthenticators::resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured = true) const +{ + std::lock_guard lock{mutex}; + + const auto token = String(credentials.getToken()); + + if (jwt_validators.empty() && throw_not_configured) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + + for (const auto & it : jwt_validators) + { + String username; + if (it.second->validate("", token, username)) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + LOG_TRACE(getLogger("JWTAuthentication"), "Extracted username {} from JWT by {}", username, it.first); + return true; + } + LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkJWTClaims(const String & claims, const TokenCredentials & credentials) const +{ + std::lock_guard lock{mutex}; + + const auto token = String(credentials.getToken()); + + if (jwt_validators.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + + for (const auto & it : jwt_validators) + { + String username; + if (it.second->validate(claims, token, username)) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + LOG_DEBUG(getLogger("JWTAuthentication"), "Authenticated with JWT for {} by {}", username, it.first); + return true; + } + LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials & credentials) const +{ + std::lock_guard lock{mutex}; + + if (access_token_processors.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + + /// lookup token in local cache if not expired. + auto cached_entry_iter = access_token_cache.find(credentials.getToken()); + if (cached_entry_iter != access_token_cache.end()) + { + if (cached_entry_iter->second.expires_at <= std::chrono::system_clock::now()) + { + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} expired, removing", cached_entry_iter->second.user_name); + access_token_cache.erase(cached_entry_iter); + } + else + { + const auto & user_data = cached_entry_iter->second; + const_cast(credentials).setUserName(user_data.user_name); + const_cast(credentials).setGroups(user_data.external_roles); + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} found, using it to authenticate", cached_entry_iter->second.user_name); + return true; + } + } + + for (const auto & it : access_token_processors) + { + if (it.second->resolveAndValidate(credentials)) + { + AccessTokenCacheEntry cache_entry; + cache_entry.user_name = credentials.getUserName(); + cache_entry.external_roles = credentials.getGroups(); + + auto default_expiration_ts = std::chrono::system_clock::now() + + std::chrono::minutes(it.second->getCacheInvalidationInterval()); + + if (credentials.getExpiresAt().has_value()) + { + if (credentials.getExpiresAt().value() < default_expiration_ts) + cache_entry.expires_at = credentials.getExpiresAt().value(); + } + else + { + cache_entry.expires_at = default_expiration_ts; + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} added", cache_entry.user_name); + + access_token_cache[credentials.getToken()] = cache_entry; + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), it.first); + return true; + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const +{ + std::lock_guard lock{mutex}; + + if (access_token_processors.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + + for (const auto & it : access_token_processors) + { + if (name == it.second->getName()) + { + if (it.second->resolveAndValidate(credentials)) { + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", + credentials.getUserName(), it.first); + return true; + } else + { + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by processor {}", name); + return false; + } + } + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token: no processor with name {}", name); + return false; +} + bool ExternalAuthenticators::checkHTTPBasicCredentials( const String & server, const BasicCredentials & credentials, const ClientInfo & client_info, SettingsChanges & settings) const { diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 6aa26bb3842a..311765560341 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -1,8 +1,10 @@ #pragma once +#include #include #include #include +#include #include #include #include @@ -13,6 +15,7 @@ #include #include +#include #include #include #include @@ -32,6 +35,7 @@ namespace DB { class SettingsChanges; +class AccessControl; class ExternalAuthenticators { @@ -45,8 +49,16 @@ class ExternalAuthenticators bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; bool checkHTTPBasicCredentials(const String & server, const BasicCredentials & credentials, const ClientInfo & client_info, SettingsChanges & settings) const; + bool resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured) const; + bool checkJWTClaims(const String & claims, const TokenCredentials & credentials) const; + + bool checkAccessTokenCredentials(const TokenCredentials & credentials) const; + bool checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const; + GSSAcceptorContext::Params getKerberosParams() const; + bool isJWTAllowed() const; + private: HTTPAuthClientParams getHTTPAuthenticationParams(const String& server) const; @@ -57,15 +69,27 @@ class ExternalAuthenticators LDAPClient::SearchResultsList last_successful_role_search_results; }; + struct AccessTokenCacheEntry + { + std::chrono::system_clock::time_point expires_at; + String user_name; + std::set external_roles; + }; + using LDAPCache = std::unordered_map; // user name -> cache entry using LDAPCaches = std::map; // server name -> cache using LDAPParams = std::map; // server name -> params + using AccessTokenCache = std::unordered_map; // Access token -> cache entry + mutable std::mutex mutex; LDAPParams ldap_client_params_blueprint TSA_GUARDED_BY(mutex) ; mutable LDAPCaches ldap_caches TSA_GUARDED_BY(mutex) ; + mutable AccessTokenCache access_token_cache TSA_GUARDED_BY(mutex) ; std::optional kerberos_params TSA_GUARDED_BY(mutex) ; std::unordered_map http_auth_servers TSA_GUARDED_BY(mutex) ; + std::unordered_map> jwt_validators TSA_GUARDED_BY(mutex) ; + std::unordered_map> access_token_processors TSA_GUARDED_BY(mutex) ; void resetImpl() TSA_REQUIRES(mutex); }; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 59335a66382c..4a5664e5a3ea 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "Access/Common/AuthenticationType.h" #include #include #include @@ -33,6 +34,7 @@ namespace ErrorCodes extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; extern const int ACCESS_STORAGE_DOESNT_ALLOW_BACKUP; + extern const int AUTHENTICATION_FAILED; extern const int WRONG_PASSWORD; extern const int IP_ADDRESS_NOT_ALLOWED; extern const int LOGICAL_ERROR; @@ -538,6 +540,9 @@ std::optional IAccessStorage::authenticateImpl( bool allow_no_password, bool allow_plaintext_password) const { + if (typeid_cast(&credentials) && !typeid_cast(&credentials)->isReady()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Could not resolve username from token"); + if (auto id = find(credentials.getUserName())) { if (auto user = tryRead(*id)) diff --git a/src/Access/JWTValidator.cpp b/src/Access/JWTValidator.cpp new file mode 100644 index 000000000000..3b9763448f05 --- /dev/null +++ b/src/Access/JWTValidator.cpp @@ -0,0 +1,367 @@ +#include "JWTValidator.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +namespace +{ + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path); +bool check_claims(const picojson::value::object & claims, const picojson::value::object & payload, const String & path) +{ + for (const auto & it : claims) + { + const auto & payload_it = payload.find(it.first); + if (payload_it == payload.end()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "Key '{}.{}' not found in JWT payload", path, it.first); + return false; + } + if (!check_claims(it.second, payload_it->second, path + "." + it.first)) + { + return false; + } + } + return true; +} + +bool check_claims(const picojson::value::array & claims, const picojson::value::array & payload, const String & path) +{ + if (claims.size() > payload.size()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload too small for claims key '{}'", path); + return false; + } + for (size_t claims_i = 0; claims_i < claims.size(); ++claims_i) + { + bool found = false; + const auto & claims_val = claims.at(claims_i); + for (const auto & payload_val : payload) + { + if (!check_claims(claims_val, payload_val, path + "[" + std::to_string(claims_i) + "]")) + continue; + found = true; + } + if (!found) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not contain an object matching claims key '{}[{}]'", path, claims_i); + return false; + } + } + return true; +} + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path) +{ + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'array' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'object' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'bool' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'double' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'std::string' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + #ifdef PICOJSON_USE_INT64 + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'int64_t' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in claims '{}'. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + #endif + LOG_ERROR(getLogger("JWTAuthentication"), "JWT claim '{}' does not match any known type", path); + return false; +} + +bool check_claims(const String & claims, const picojson::value::object & payload) +{ + if (claims.empty()) + return true; + picojson::value json; + auto errors = picojson::parse(json, claims); + if (!errors.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: {}", errors); + if (!json.is()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: is not an object"); + return check_claims(json.get(), payload, ""); +} + +} + +bool IJWTValidator::validate(const String & claims, const String & token, String & username) +{ + try + { + auto decoded_jwt = jwt::decode(token); + + validateImpl(decoded_jwt); + + if (!check_claims(claims, decoded_jwt.get_payload_json())) + return false; + + username = decoded_jwt.get_subject(); + + return true; + } + catch (const std::exception & ex) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: Failed to validate JWT: {}", name, ex.what()); + return false; + } +} + +void SimpleJWTValidatorParams::validate() const +{ + if (algo == "ps256" || + algo == "ps384" || + algo == "ps512" || + algo == "ed25519" || + algo == "ed448" || + algo == "rs256" || + algo == "rs384" || + algo == "rs512" || + algo == "es256" || + algo == "es256k" || + algo == "es384" || + algo == "es512" ) + { + if (public_key.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `public_key` parameter required for {}", algo); + } + else if (algo == "hs256" || + algo == "hs384" || + algo == "hs512" ) + { + if (static_key.empty()) + throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `static_key` parameter required for {}", algo); + } + else if (algo != "none") + throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); +} + + +SimpleJWTValidator::SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_) + : IJWTValidator(name_), verifier(jwt::verify()) +{ + auto algo = params_.algo; + + if (algo == "none") + verifier = verifier.allow_algorithm(jwt::algorithm::none()); + else if (algo == "ps256") + verifier = verifier.allow_algorithm(jwt::algorithm::ps256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ps384") + verifier = verifier.allow_algorithm(jwt::algorithm::ps384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ps512") + verifier = verifier.allow_algorithm(jwt::algorithm::ps512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ed25519") + verifier = verifier.allow_algorithm(jwt::algorithm::ed25519(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ed448") + verifier = verifier.allow_algorithm(jwt::algorithm::ed448(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es256") + verifier = verifier.allow_algorithm(jwt::algorithm::es256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es256k") + verifier = verifier.allow_algorithm(jwt::algorithm::es256k(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es384") + verifier = verifier.allow_algorithm(jwt::algorithm::es384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es512") + verifier = verifier.allow_algorithm(jwt::algorithm::es512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo.starts_with("hs")) + { + auto key = params_.static_key; + if (params_.static_key_in_base64) + key = base64Decode(key); + if (algo == "hs256") + verifier = verifier.allow_algorithm(jwt::algorithm::hs256(key)); + else if (algo == "hs384") + verifier = verifier.allow_algorithm(jwt::algorithm::hs384(key)); + else if (algo == "hs512") + verifier = verifier.allow_algorithm(jwt::algorithm::hs512(key)); + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); + } + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); +} + +void SimpleJWTValidator::validateImpl(const jwt::decoded_jwt & token) const +{ + verifier.verify(token); +} + +void JWKSValidator::validateImpl(const jwt::decoded_jwt & token) const +{ + auto jwk = provider->getJWKS().get_jwk(token.get_key_id()); + auto subject = token.get_subject(); + auto algo = Poco::toLower(token.get_algorithm()); + auto verifier = jwt::verify(); + String public_key; + + try + { + auto issuer = token.get_issuer(); + auto x5c = jwk.get_x5c_key_value(); + + if (!x5c.empty() && !issuer.empty()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: Verifying {} with 'x5c' key", name, subject); + public_key = jwt::helper::convert_base64_der_to_pem(x5c); + } + } + catch (const jwt::error::claim_not_present_exception &) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: issuer or x5c was not specified, skip verification against them", name); + } + catch (const std::bad_cast &) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: invalid claim value type found, claims must be strings"); + } + + if (public_key.empty()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: `issuer` or `x5c` not present, verifying {} with RSA components", name, subject); + const auto modulus = jwk.get_jwk_claim("n").as_string(); + const auto exponent = jwk.get_jwk_claim("e").as_string(); + public_key = jwt::helper::create_public_key_from_rsa_components(modulus, exponent); + } + + if (jwk.has_algorithm() && Poco::toLower(jwk.get_algorithm()) != algo) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT validation error: `alg` in JWK does not match the algorithm used in JWT"); + + if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(public_key, "", "", "")); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(public_key, "", "", "")); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(public_key, "", "", "")); + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); + + verifier = verifier.leeway(60UL); + verifier.verify(token); +} + + +std::unique_ptr IJWTValidator::parseJWTValidator( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name) +{ + if (config.hasProperty(prefix + ".algo")) + { + SimpleJWTValidatorParams params = {}; + params.algo = Poco::toLower(config.getString(prefix + ".algo")); + params.static_key = config.getString(prefix + ".static_key", ""); + params.static_key_in_base64 = config.getBool(prefix + ".static_key_in_base64", false); + params.public_key = config.getString(prefix + ".public_key", ""); + params.private_key = config.getString(prefix + ".private_key", ""); + params.public_key_password = config.getString(prefix + ".public_key_password", ""); + params.private_key_password = config.getString(prefix + ".private_key_password", ""); + params.validate(); + return std::make_unique(name, params); + } + + std::shared_ptr provider; + if (config.hasProperty(prefix + ".uri")) + { + provider = std::make_shared(config.getString(prefix + ".uri"), config.getInt(prefix + ".refresh_ms", 300000)); + } + else if (config.hasProperty(prefix + ".static_jwks") || config.hasProperty(prefix + ".static_jwks_file")) + { + StaticJWKSParams params{ + config.getString(prefix + ".static_jwks", ""), + config.getString(prefix + ".static_jwks_file", "") + }; + provider = std::make_shared(params); + } + else + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Either JWKS or JWKS URI must be specified in configuration"); + + return std::make_unique(name, provider); +} + +} diff --git a/src/Access/JWTValidator.h b/src/Access/JWTValidator.h new file mode 100644 index 000000000000..03bcdf4990d1 --- /dev/null +++ b/src/Access/JWTValidator.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include + +#include "Access/HTTPAuthClient.h" + +#include +#include +#include + +namespace DB +{ + +class IJWTValidator +{ +public: + explicit IJWTValidator(const String & name_) : name(name_) {} + virtual bool validate(const String & claims, const String & token, String & username); + virtual ~IJWTValidator() = default; + + static std::unique_ptr parseJWTValidator( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name); + +protected: + virtual void validateImpl(const jwt::decoded_jwt & token) const = 0; + const String name; +}; + +struct SimpleJWTValidatorParams +{ + String algo; + String static_key; + bool static_key_in_base64; + String public_key; + String private_key; + String public_key_password; + String private_key_password; + void validate() const; +}; + +class SimpleJWTValidator : public IJWTValidator +{ +public: + explicit SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_); +private: + void validateImpl(const jwt::decoded_jwt & token) const override; + jwt::verifier verifier; +}; + +class JWKSValidator : public IJWTValidator +{ +public: + explicit JWKSValidator(const String & name_, std::shared_ptr provider_) + : IJWTValidator(name_), provider(provider_) {} +private: + void validateImpl(const jwt::decoded_jwt & token) const override; + + std::shared_ptr provider; +}; +} diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp new file mode 100644 index 000000000000..baccf3489815 --- /dev/null +++ b/src/Access/TokenAccessStorage.cpp @@ -0,0 +1,396 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +TokenAccessStorage::TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) + : IAccessStorage(storage_name_), access_control(access_control_), config(config_), prefix(prefix_), + memory_storage(storage_name_, access_control.getChangesNotifier(), false) +{ + setConfiguration(); +} + +void TokenAccessStorage::setConfiguration() +{ + std::lock_guard lock(mutex); + + const String prefix_str = (prefix.empty() ? "" : prefix + "."); + + provider_name = config.getString(prefix_str + "processor"); + if (provider_name.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "'processor' must be specified for Token user directory"); + + const bool has_roles = config.has(prefix_str + "roles"); + + std::set common_roles_cfg; + if (has_roles) + { + Poco::Util::AbstractConfiguration::Keys role_names; + config.keys(prefix_str + "roles", role_names); + + common_roles_cfg.insert(role_names.begin(), role_names.end()); + } + + common_role_names.swap(common_roles_cfg); + + external_role_hashes.clear(); + users_per_roles.clear(); + roles_per_users.clear(); + granted_role_names.clear(); + granted_role_ids.clear(); + + role_change_subscription = access_control.subscribeForChanges( + [this] (const UUID & id, const AccessEntityPtr & entity) + { + this->processRoleChange(id, entity); + } + ); +} + +void TokenAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name) +{ + std::vector user_ids; + + // Build a list of ids of the relevant users. + if (common_role_names.contains(role_name)) + { + user_ids = memory_storage.findAll(); + } + else + { + const auto it = users_per_roles.find(role_name); + if (it != users_per_roles.end()) + { + const auto & user_names = it->second; + user_ids.reserve(user_names.size()); + + for (const auto & user_name : user_names) + { + if (const auto user_id = memory_storage.find(user_name)) + user_ids.emplace_back(*user_id); + } + } + } + + // Update the granted roles of the relevant users. + if (!user_ids.empty()) + { + auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + if (grant) + changed_user->granted_roles.grant(role_id); + else + changed_user->granted_roles.revoke(role_id); + return changed_user; + } + return entity_; + }; + + memory_storage.update(user_ids, update_func); + } + + // Actualize granted_role_* mappings. + if (grant) + { + if (!user_ids.empty()) + { + granted_role_names.insert_or_assign(role_id, role_name); + granted_role_ids.insert_or_assign(role_name, role_id); + } + } + else + { + granted_role_ids.erase(role_name); + granted_role_names.erase(role_id); + } +} + +void TokenAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) +{ + std::lock_guard lock(mutex); + const auto role = typeid_cast>(entity); + const auto it = granted_role_names.find(id); + + if (role) // Added or renamed a role. + { + const auto & new_role_name = role->getName(); + if (it != granted_role_names.end()) // Renamed a granted role. + { + const auto & old_role_name = it->second; + if (new_role_name != old_role_name) + { + // Revoke the old role first, then grant the new role. + applyRoleChangeNoLock(false /* revoke */, id, old_role_name); + applyRoleChangeNoLock(true /* grant */, id, new_role_name); + } + } + else // Added a role. + { + applyRoleChangeNoLock(true /* grant */, id, new_role_name); + } + } + else // Removed a role. + { + if (it != granted_role_names.end()) // Removed a granted role. + { + const auto & old_role_name = it->second; + applyRoleChangeNoLock(false /* revoke */, id, old_role_name); + } + } +} + +const char * TokenAccessStorage::getStorageType() const +{ + return STORAGE_TYPE; +} + +bool TokenAccessStorage::exists(const UUID & id) const +{ + std::lock_guard lock(mutex); + return memory_storage.exists(id); +} + +String TokenAccessStorage::getStorageParamsJSON() const +{ + std::lock_guard lock(mutex); + Poco::JSON::Object params_json; + + params_json.set("provider", provider_name); + + Poco::JSON::Array common_role_names_json; + for (const auto & role : common_role_names) + { + common_role_names_json.add(role); + } + params_json.set("roles", common_role_names_json); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(params_json, oss); + + return oss.str(); +} + +bool TokenAccessStorage::areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const +{ + if (!credentials.isReady()) + return false; + + if (credentials.getUserName() != user.getName()) + return false; + + if (const auto * token_credentials = dynamic_cast(&credentials)) + return external_authenticators.checkAccessTokenCredentials(*token_credentials); + + return false; +} + +std::optional TokenAccessStorage::findImpl(AccessEntityType type, const String & name) const +{ + std::lock_guard lock(mutex); + return memory_storage.find(type, name); +} + + +std::vector TokenAccessStorage::findAllImpl(AccessEntityType type) const +{ + std::lock_guard lock(mutex); + return memory_storage.findAll(type); +} + +AccessEntityPtr TokenAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const +{ + std::lock_guard lock(mutex); + return memory_storage.read(id, throw_if_not_exists); +} + +std::optional> TokenAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const +{ + std::lock_guard lock(mutex); + return memory_storage.readNameWithType(id, throw_if_not_exists); +} + +void TokenAccessStorage::assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const +{ + const auto & user_name = user.getName(); + auto & granted_roles = user.granted_roles; + + auto grant_role = [this, &user_name, &granted_roles] (const String & role_name, const bool common) + { + auto it = granted_role_ids.find(role_name); + if (it == granted_role_ids.end()) + { + if (const auto role_id = access_control.find(role_name)) + { + granted_role_names.insert_or_assign(*role_id, role_name); + it = granted_role_ids.insert_or_assign(role_name, *role_id).first; + } + } + + if (it != granted_role_ids.end()) + { + const auto & role_id = it->second; + granted_roles.grant(role_id); + } + else + { + LOG_TRACE(getLogger(), "Did not grant {} role '{}' to user '{}': role not found", (common ? "common" : "mapped"), role_name, user_name); + } + }; + + external_role_hashes.erase(user_name); + granted_roles = {}; + const auto old_role_names = std::move(roles_per_users[user_name]); + + // Grant the common roles first. + for (const auto & role_name : common_role_names) + { + grant_role(role_name, true /* common */); + } + + // Grant the mapped external roles and actualize users_per_roles mapping. + // external_roles allowed to overlap with common_role_names. + for (const auto & role_name : external_roles) + { + grant_role(role_name, false /* mapped */); + users_per_roles[role_name].insert(user_name); + } + + // Cleanup users_per_roles and granted_role_* mappings. + for (const auto & old_role_name : old_role_names) + { + if (external_roles.contains(old_role_name)) + continue; + + const auto rit = users_per_roles.find(old_role_name); + if (rit == users_per_roles.end()) + continue; + + auto & user_names = rit->second; + user_names.erase(user_name); + + if (!user_names.empty()) + continue; + + users_per_roles.erase(rit); + + if (common_role_names.contains(old_role_name)) + continue; + + const auto iit = granted_role_ids.find(old_role_name); + if (iit == granted_role_ids.end()) + continue; + + const auto old_role_id = iit->second; + granted_role_names.erase(old_role_id); + granted_role_ids.erase(iit); + } + + // Actualize roles_per_users mapping and external_role_hashes cache. + if (external_roles.empty()) + roles_per_users.erase(user_name); + else + roles_per_users[user_name] = std::move(external_roles); + + external_role_hashes[user_name] = external_roles_hash; +} + +void TokenAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const +{ + // No need to include common_role_names in this hash each time, since they don't change. + const auto external_roles_hash = boost::hash>{}(external_roles); + + // Map and grant the roles from scratch only if the list of external role has changed. + const auto it = external_role_hashes.find(user_name); + if (it != external_role_hashes.end() && it->second == external_roles_hash) + return; + + auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + assignRolesNoLock(*changed_user, external_roles, external_roles_hash); + return changed_user; + } + return entity_; + }; + + memory_storage.update(id, update_func); +} + + +std::optional TokenAccessStorage::authenticateImpl( + const Credentials & credentials, + const Poco::Net::IPAddress & address, + [[maybe_unused]] const ExternalAuthenticators & external_authenticators, + [[maybe_unused]] bool throw_if_user_not_exists, + bool /* allow_no_password */, + bool /* allow_plaintext_password */) const +{ + std::lock_guard lock(mutex); + auto id = memory_storage.find(credentials.getUserName()); + UserPtr user = id ? memory_storage.read(*id) : nullptr; + + const auto & token_credentials = typeid_cast(credentials); + + if (!external_authenticators.checkAccessTokenCredentialsByExactProcessor(token_credentials, provider_name)) + { + // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. + if (throw_if_user_not_exists) + throwNotFound(AccessEntityType::USER, credentials.getUserName()); + else + return {}; + } + + std::shared_ptr new_user; + if (!user) + { + // User does not exist, so we create one, and will add it if authentication is successful. + new_user = std::make_shared(); + new_user->setName(credentials.getUserName()); + new_user->authentication_methods.emplace_back(AuthenticationType::JWT); + user = new_user; + } + + if (!isAddressAllowed(*user, address)) + throwAddressNotAllowed(address); + + std::set external_roles = token_credentials.getGroups(); + + if (new_user) + { + assignRolesNoLock(*new_user, external_roles, boost::hash>{}(external_roles)); + id = memory_storage.insert(new_user); + } + else + { + // Just in case external_roles are changed. + updateAssignedRolesNoLock(*id, user->getName(), external_roles); + } + + if (id) + return AuthResult{ .user_id = *id, .authentication_data = AuthenticationData(AuthenticationType::JWT) }; + return std::nullopt; +} + + +} diff --git a/src/Access/TokenAccessStorage.h b/src/Access/TokenAccessStorage.h new file mode 100644 index 000000000000..0908f30ea519 --- /dev/null +++ b/src/Access/TokenAccessStorage.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace Poco +{ + namespace Util + { + class AbstractConfiguration; + } +} + + +namespace DB +{ +class AccessControl; + +/// Implementation of IAccessStorage which allows to import user data from oauth server using access token. +/// Normally, this should be unified with LDAPAccessStorage, but not done to minimize changes to code that is common with upstream. +class TokenAccessStorage : public IAccessStorage +{ +public: + static constexpr char STORAGE_TYPE[] = "token"; + + explicit TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix); + ~TokenAccessStorage() override = default; + + // IAccessStorage implementations. + const char * getStorageType() const override; + String getStorageParamsJSON() const override; + bool isReadOnly() const override { return true; } + bool exists(const UUID & id) const override; + +private: // IAccessStorage implementations. + + mutable std::recursive_mutex mutex; // Note: Reentrance possible by internal role lookup via access_control + AccessControl & access_control; + const Poco::Util::AbstractConfiguration & config; + const String & prefix; + + String provider_name; + + std::set common_role_names; // role name that should be granted to all users at all times + mutable std::map external_role_hashes; + mutable std::map> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles) + mutable std::map> roles_per_users; // user name -> role names (...that should be granted to it; may but don't have to include common roles) + mutable std::map granted_role_names; // (currently granted) role id -> its name + mutable std::map granted_role_ids; // (currently granted) role name -> its id + scope_guard role_change_subscription; + mutable MemoryAccessStorage memory_storage; + + void setConfiguration(); + void processRoleChange(const UUID & id, const AccessEntityPtr & entity); + + bool areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; + + std::optional findImpl(AccessEntityType type, const String & name) const override; + std::vector findAllImpl(AccessEntityType type) const override; + AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; + std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override; + std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, + [[maybe_unused]] const ExternalAuthenticators & external_authenticators, + [[maybe_unused]] bool throw_if_user_not_exists, + bool allow_no_password, bool allow_plaintext_password) const override; + + + void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name); + void assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const; + void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const; +}; +} diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index fec9db829eb7..e0cab5b337e8 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -13,6 +13,7 @@ #include #include #include +#include "Access/Credentials.h" #include #include #include @@ -131,6 +132,7 @@ namespace bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex"); bool has_ldap = config.has(user_config + ".ldap"); bool has_kerberos = config.has(user_config + ".kerberos"); + bool has_jwt = config.has(user_config + ".jwt"); const auto certificates_config = user_config + ".ssl_certificates"; bool has_certificates = config.has(certificates_config); @@ -142,18 +144,18 @@ namespace bool has_http_auth = config.has(http_auth_config); size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex - + has_ldap + has_kerberos + has_certificates + has_ssh_keys + has_http_auth + has_scram_password_sha256_hex; + + has_ldap + has_kerberos + has_certificates + has_ssh_keys + has_http_auth + has_scram_password_sha256_hex + has_jwt; if (num_password_fields > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "More than one field of 'password', 'password_sha256_hex', " "'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'ssl_certificates', 'ssh_keys', " - "'http_authentication' are used to specify authentication info for user {}. " + "'http_authentication', 'jwt' are used to specify authentication info for user {}. " "Must be only one of them.", user_name); if (num_password_fields < 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Either 'password' or 'password_sha256_hex' " "or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos " - "or 'ssl_certificates' or 'ssh_keys' or 'http_authentication' must be specified for user {}.", user_name); + "or 'ssl_certificates' or 'ssh_keys' or 'http_authentication' or 'jwt' must be specified for user {}.", user_name); if (has_password_plaintext) { @@ -277,6 +279,10 @@ namespace auto scheme = config.getString(http_auth_config + ".scheme"); user->authentication_methods.back().setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme)); } + else if (has_jwt) + { + user->authentication_methods.emplace_back(AuthenticationType::JWT); + } else { user->authentication_methods.emplace_back(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f7a3cf851b1..de0a3b013f2d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -394,6 +394,7 @@ target_link_libraries(clickhouse_common_io ch_contrib::zlib pcg_random Poco::Foundation + ch_contrib::jwt-cpp ) if (TARGET ch_contrib::libfiu) diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 346d9800da11..a514cd3e64a2 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -116,8 +116,11 @@ void ASTAuthenticationData::formatImpl(WriteBuffer & ostr, const FormatSettings } case AuthenticationType::JWT: { - prefix = "CLAIMS"; - parameter = true; + if (!children.empty()) + { + prefix = "CLAIMS"; + parameter = true; + } break; } case AuthenticationType::LDAP: diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index ff1ec4f12705..ae0485773489 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -17,7 +17,7 @@ class ASTAuthenticationData; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']|{WITH jwt [CLAIMS 'json_object']}}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [DEFAULT DATABASE database | NONE] @@ -26,7 +26,7 @@ class ASTAuthenticationData; * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']|{WITH jwt [CLAIMS 'json_object']}}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [DEFAULT DATABASE database | NONE] diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 4f2c9c4e983d..c1adbba0b47b 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -75,6 +75,7 @@ namespace bool expect_ssl_cert_subjects = false; bool expect_public_ssh_key = false; bool expect_http_auth_server = false; + bool expect_claims = false; auto parse_non_password_based_type = [&](auto check_type) { @@ -92,6 +93,8 @@ namespace expect_public_ssh_key = true; else if (check_type == AuthenticationType::HTTP) expect_http_auth_server = true; + else if (check_type == AuthenticationType::JWT) + expect_claims = true; else if (check_type != AuthenticationType::NO_PASSWORD) expect_password = true; @@ -152,6 +155,7 @@ namespace ASTPtr http_auth_scheme; ASTPtr ssl_cert_subjects; std::optional ssl_cert_subject_type; + ASTPtr jwt_claims; if (expect_password || expect_hash) { @@ -216,6 +220,14 @@ namespace return false; } } + else if (expect_claims) + { + if (ParserKeyword{Keyword::CLAIMS}.ignore(pos, expected)) + { + if (!ParserStringAndSubstitution{}.parse(pos, jwt_claims, expected)) + return false; + } + } auth_data = std::make_shared(); @@ -241,6 +253,9 @@ namespace if (http_auth_scheme) auth_data->children.push_back(std::move(http_auth_scheme)); + if (jwt_claims) + auth_data->children.push_back(std::move(jwt_claims)); + parseValidUntil(pos, expected, auth_data->valid_until); return true; diff --git a/src/Parsers/Access/ParserCreateUserQuery.h b/src/Parsers/Access/ParserCreateUserQuery.h index 4dfff8713d76..5f4cfcd6c45f 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.h +++ b/src/Parsers/Access/ParserCreateUserQuery.h @@ -7,7 +7,7 @@ namespace DB { /** Parses queries like * CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}|{WITH jwt}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] @@ -15,7 +15,7 @@ namespace DB * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}|{WITH jwt}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index e4db7beb9d4e..9f363570ddc2 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -85,6 +85,7 @@ namespace DB MR_MACROS(CHECK_TABLE, "CHECK TABLE") \ MR_MACROS(CHECK_GRANT, "CHECK GRANT") \ MR_MACROS(CHECK, "CHECK") \ + MR_MACROS(CLAIMS, "CLAIMS") \ MR_MACROS(CLEANUP, "CLEANUP") \ MR_MACROS(CLEAR_COLUMN, "CLEAR COLUMN") \ MR_MACROS(CLEAR_INDEX, "CLEAR INDEX") \ diff --git a/src/Server/HTTP/authenticateUserByHTTP.cpp b/src/Server/HTTP/authenticateUserByHTTP.cpp index c956ef4489c9..07942ef43ab6 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.cpp +++ b/src/Server/HTTP/authenticateUserByHTTP.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -16,7 +17,7 @@ # include #endif - +const String BEARER_PREFIX = "bearer "; namespace DB { @@ -77,6 +78,8 @@ bool authenticateUserByHTTP( bool has_http_credentials = request.hasCredentials() && request.get("Authorization") != "never"; bool has_credentials_in_query_params = params.has("user") || params.has("password"); + std::string jwt_token = request.get("X-ClickHouse-JWT-Token", request.get("Authorization", (params.has("token") ? BEARER_PREFIX + params.get("token") : ""))); + std::string spnego_challenge; #if USE_SSL X509Certificate::Subjects certificate_subjects; @@ -155,7 +158,7 @@ bool authenticateUserByHTTP( if (spnego_challenge.empty()) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: SPNEGO challenge is empty"); } - else + else if (Poco::icompare(scheme, "Bearer") < 0) { throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: '{}' HTTP Authorization scheme is not supported", scheme); } @@ -212,6 +215,17 @@ bool authenticateUserByHTTP( } } #endif + else if (!jwt_token.empty() && Poco::toLower(jwt_token).starts_with(BEARER_PREFIX)) + { + const auto token_credentials = TokenCredentials(jwt_token.substr(BEARER_PREFIX.length())); + const auto & external_authenticators = global_context->getAccessControl().getExternalAuthenticators(); + + if (!external_authenticators.resolveJWTCredentials(token_credentials, false) + && !external_authenticators.checkAccessTokenCredentials(token_credentials)) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: Token could not be verified."); + + current_credentials = std::make_unique(token_credentials); + } else // I.e., now using user name and password strings ("Basic"). { if (!current_credentials) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 6b16465db7bf..d321bee2a7b4 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -1767,6 +1768,10 @@ void TCPHandler::receiveHello() if (is_ssh_based_auth) user.erase(0, std::string_view(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER).size()); + is_jwt_based_auth = user.starts_with(EncodedUserInfo::JWT_AUTHENTICAION_MARKER); + if (is_jwt_based_auth) + user.erase(0, std::string_view(EncodedUserInfo::JWT_AUTHENTICAION_MARKER).size()); + session = makeSession(); const auto & client_info = session->getClientInfo(); @@ -1854,6 +1859,19 @@ void TCPHandler::receiveHello() } #endif + if (is_jwt_based_auth) + { + auto credentials = TokenCredentials(password); + + const auto & external_authenticators = server.context()->getAccessControl().getExternalAuthenticators(); + + if (!external_authenticators.resolveJWTCredentials(credentials, false)) + external_authenticators.checkAccessTokenCredentials(credentials); + + session->authenticate(credentials, getClientAddress(client_info)); + return; + } + session->authenticate(user, password, getClientAddress(client_info)); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index ed12f5db79ca..862a8305f94e 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -235,6 +235,7 @@ class TCPHandler : public Poco::Net::TCPServerConnection String default_database; bool is_ssh_based_auth = false; /// authentication is via SSH pub-key challenge + bool is_jwt_based_auth = false; /// authentication is via JWT /// For inter-server secret (remote_server.*.secret) bool is_interserver_mode = false; bool is_interserver_authenticated = false; diff --git a/tests/integration/test_jwt_auth/__init__.py b/tests/integration/test_jwt_auth/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_jwt_auth/configs/users.xml b/tests/integration/test_jwt_auth/configs/users.xml new file mode 100644 index 000000000000..b3d3372ebaa9 --- /dev/null +++ b/tests/integration/test_jwt_auth/configs/users.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + default + default + + + diff --git a/tests/integration/test_jwt_auth/configs/validators.xml b/tests/integration/test_jwt_auth/configs/validators.xml new file mode 100644 index 000000000000..1522937629cd --- /dev/null +++ b/tests/integration/test_jwt_auth/configs/validators.xml @@ -0,0 +1,24 @@ + + + + + HS256 + my_secret + false + + + + hs256 + other_secret + false + + + + {"keys": [{"kty": "RSA", "alg": "rs256", "kid": "mykid", "n": "lICGC8S5pObyASih5qfmwuclG0oKsbzY2z9vgwqyhTYQOWcqYcTjVV4aQ30qb6E0-5W6rJ-jx9zx6GuAEGMiG_aWJEdbUAMGp-L1kz4lrw5U6GlwoZIvk4wqoRwsiyc-mnDMQAmiZLBNyt3wU6YnKgYmb4O1cSzcZ5HMbImJpj4tpYjqnIazvYMn_9Pxjkl0ezLCr52av0UkWHro1H4QMVfuEoNmHuWPww9jgHn-I-La0xdOhRpAa0XnJi65dXZd4330uWjeJwt413yz881uS4n1OLOGKG8ImDcNlwU_guyvk0n0aqT0zkOAPp9_yYo13MPWmiRCfOX8ozdN7VDIJw", "e": "AQAB"}]} + + + + http://resolver:8080/.well-known/jwks.json + + + diff --git a/tests/integration/test_jwt_auth/helpers/generate_private_key.py b/tests/integration/test_jwt_auth/helpers/generate_private_key.py new file mode 100644 index 000000000000..7b54fa63368b --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/generate_private_key.py @@ -0,0 +1,21 @@ +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend + +# Generate RSA private key +private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, # Key size of 2048 bits + backend=default_backend() +) + +# Save the private key to a PEM file +pem_private_key = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() # You can add encryption if needed +) + +# Write the private key to a file +with open("new_private_key", "wb") as pem_file: + pem_file.write(pem_private_key) diff --git a/tests/integration/test_jwt_auth/helpers/jwt_jwk.py b/tests/integration/test_jwt_auth/helpers/jwt_jwk.py new file mode 100644 index 000000000000..265882efce76 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/jwt_jwk.py @@ -0,0 +1,113 @@ +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + +import base64 +import json +import jwt + + +""" +Only RS* family algorithms are supported!!! +""" +with open("./private_key_2", "rb") as key_file: + private_key = serialization.load_pem_private_key( + key_file.read(), + password=None, + ) + + +public_key = private_key.public_key() + + +def to_base64_url(data): + return base64.urlsafe_b64encode(data).decode("utf-8").rstrip("=") + + +def rsa_key_to_jwk(private_key=None, public_key=None): + if private_key: + # Convert the private key to its components + private_numbers = private_key.private_numbers() + public_numbers = private_key.public_key().public_numbers() + + jwk = { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": to_base64_url( + public_numbers.n.to_bytes( + (public_numbers.n.bit_length() + 7) // 8, byteorder="big" + ) + ), + "e": to_base64_url( + public_numbers.e.to_bytes( + (public_numbers.e.bit_length() + 7) // 8, byteorder="big" + ) + ), + "d": to_base64_url( + private_numbers.d.to_bytes( + (private_numbers.d.bit_length() + 7) // 8, byteorder="big" + ) + ), + "p": to_base64_url( + private_numbers.p.to_bytes( + (private_numbers.p.bit_length() + 7) // 8, byteorder="big" + ) + ), + "q": to_base64_url( + private_numbers.q.to_bytes( + (private_numbers.q.bit_length() + 7) // 8, byteorder="big" + ) + ), + "dp": to_base64_url( + private_numbers.dmp1.to_bytes( + (private_numbers.dmp1.bit_length() + 7) // 8, byteorder="big" + ) + ), + "dq": to_base64_url( + private_numbers.dmq1.to_bytes( + (private_numbers.dmq1.bit_length() + 7) // 8, byteorder="big" + ) + ), + "qi": to_base64_url( + private_numbers.iqmp.to_bytes( + (private_numbers.iqmp.bit_length() + 7) // 8, byteorder="big" + ) + ), + } + elif public_key: + # Convert the public key to its components + public_numbers = public_key.public_numbers() + + jwk = { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": to_base64_url( + public_numbers.n.to_bytes( + (public_numbers.n.bit_length() + 7) // 8, byteorder="big" + ) + ), + "e": to_base64_url( + public_numbers.e.to_bytes( + (public_numbers.e.bit_length() + 7) // 8, byteorder="big" + ) + ), + } + else: + raise ValueError("You must provide either a private or public key.") + + return jwk + + +# Convert to JWK +jwk_private = rsa_key_to_jwk(private_key=private_key) +jwk_public = rsa_key_to_jwk(public_key=public_key) + +print(f"Private JWK:\n{json.dumps(jwk_private)}\n") +print(f"Public JWK:\n{json.dumps(jwk_public)}\n") + +payload = {"sub": "jwt_user", "iss": "test_iss"} + +# Create a JWT +token = jwt.encode(payload, private_key, headers={"kid": "mykid"}, algorithm="RS512") +print(f"JWT:\n{token}") diff --git a/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py b/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py new file mode 100644 index 000000000000..5f1c7e0340af --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py @@ -0,0 +1,43 @@ +import jwt +import datetime + + +def create_jwt( + payload: dict, secret: str, algorithm: str = "HS256", expiration_minutes: int = None +) -> str: + """ + Create a JWT using a static secret and a specified encryption algorithm. + + :param payload: The payload to include in the JWT (as a dictionary). + :param secret: The secret key used to sign the JWT. + :param algorithm: The encryption algorithm to use (default is 'HS256'). + :param expiration_minutes: The time until the token expires (default is 60 minutes). + :return: The encoded JWT as a string. + """ + if expiration_minutes: + expiration = datetime.datetime.utcnow() + datetime.timedelta( + minutes=expiration_minutes + ) + payload["exp"] = expiration + + return jwt.encode(payload, secret, algorithm=algorithm) + + +if __name__ == "__main__": + secret = "my_secret" + payload = {"sub": "jwt_user"} # `sub` must contain user name + + """ + Supported algorithms: + | HMSC | RSA | ECDSA | PSS | EdDSA | + | ----- | ----- | ------ | ----- | ------- | + | HS256 | RS256 | ES256 | PS256 | Ed25519 | + | HS384 | RS384 | ES384 | PS384 | Ed448 | + | HS512 | RS512 | ES512 | PS512 | | + | | | ES256K | | | + And None + """ + algorithm = "HS256" + + token = create_jwt(payload, secret, algorithm) + print(f"Generated JWT: {token}") diff --git a/tests/integration/test_jwt_auth/helpers/private_key_1 b/tests/integration/test_jwt_auth/helpers/private_key_1 new file mode 100644 index 000000000000..a076a86e17a4 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/private_key_1 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAlICGC8S5pObyASih5qfmwuclG0oKsbzY2z9vgwqyhTYQOWcq +YcTjVV4aQ30qb6E0+5W6rJ+jx9zx6GuAEGMiG/aWJEdbUAMGp+L1kz4lrw5U6Glw +oZIvk4wqoRwsiyc+mnDMQAmiZLBNyt3wU6YnKgYmb4O1cSzcZ5HMbImJpj4tpYjq +nIazvYMn/9Pxjkl0ezLCr52av0UkWHro1H4QMVfuEoNmHuWPww9jgHn+I+La0xdO +hRpAa0XnJi65dXZd4330uWjeJwt413yz881uS4n1OLOGKG8ImDcNlwU/guyvk0n0 +aqT0zkOAPp9/yYo13MPWmiRCfOX8ozdN7VDIJwIDAQABAoIBADZfiLUuZrrWRK3f +7sfBmmCquY9wYNILT2uXooDcndjgnrgl6gK6UHKlbgBgB/WvlPK5NAyYtyMq5vgu +xEk7wvVyKC9IYUq+kOVP2JL9IlcibDxcvvypxfnETKeI5VZeHDH4MxEPdgJf+1vY +P3KhV52vestB8mFqB5l0bOEgyuGvO3/3D1JjOnFLS/K2vOj8D/KDRmwXRCcGHTxj +dj3wJH4UbCIsLgiaQBPkFmTteJDICb+7//6YQuB0t8sR/DZS9Z0GWcfy04Cp/m/E +4rRoTNz80MbbU9+k0Ly360SxPizcjpPYSRSD025i8Iqv8jvelq7Nzg69Kubc0KfN +mMrRdMECgYEAz4b7+OX+aO5o2ZQS+fHc8dyWc5umC+uT5xrUm22wZLYA5O8x0Rgj +vdO/Ho/XyN/GCyvNNV2rI2+CBTxez6NqesGDEmJ2n7TQ03xXLCVsnwVz694sPSMO +pzTbU6e42jvDo5DMPDv0Pg1CVQuM9ka6wb4DcolMyDql6QddY3iXHBkCgYEAtzAl +xEAABqdFAnCs3zRf9EZphGJiJ4gtoWmCxQs+IcrfyBNQCy6GqrzJOZ7fQiEoAeII +V0JmsNcnx3U1W0lp8N+1QNZoB4fOWXaX08BvOEe7gbJ6Xl5t52j792vQp1txpBhE +UDhz8m5R9i5qb3BzrYBiSTfak0Pq56Xw3jRDjj8CgYEAqX2QS07kQqT8gz85ZGOR +1QMY6aCks7WaXTR/kdW7K/Wts0xb/m7dugq3W+mVDh0c7UC/36b5v/4xTb9pm+HW +dB2ZxCkgwvz1VNSHiamjFhlo/Km+rcv1CsDTpHYmNi57cRowg71flFJV64l8fiN0 +IgnjXOcgC6RCnpiCQFxb5fkCgYB+Zq2YleSuspqOjXrrZPNU1YUXgN9jkbaSqwA9 +wH01ygvRvWm83XS0uSFMLhC1S7WUXwgMVdgP69YZ7glMHQMJ3wLtY0RS9eVvm8I1 +rZHQzsZWPvXqydOiGrHJzs4hvJpUdR4mEF4JCRBrAyoUDQ70yCKJjQ24EeQzxS/H +015N9wKBgB8DdFPvKXyygTMnBoZdpAhkE/x3TTi7DsLBxj7QxKmSHzlHGz0TubIB +m5/p9dGawQNzD4JwASuY5r4lKXmvYr+4TQPLq6c7EnoIZSwLdge+6PDhnDWJzvk1 +S/RuHWW4FKGzBStTmstG3m0xzxTMnQkV3kPimMim3I3VsxxeGEdq +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_jwt_auth/helpers/private_key_2 b/tests/integration/test_jwt_auth/helpers/private_key_2 new file mode 100644 index 000000000000..d0d1576f2017 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/private_key_2 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0RRsKcZ5j9UckjioG4Phvav3dkg2sXP6tQ7ug0yowAo/u2Hf +fB+1OjKuhWTpA3E3YkMKj0RrT+tuUpmZEXqCAipEV7XcfCv3o7Poa7HTq1ti/abV +wT/KyfGjoNBBSJH4LTNAyo2J8ySKSDtpAEU52iL7s40Ra6I0vqp7/aRuPF5M4zcH +zN3zarG5EfSVSG1+gTkaRv8XJbra0IeIINmKv0F4++ww8ZxXTR6cvI+MsArUiAPw +zf7s5dMR4DNRG6YNTrPA0pTOqQE9sRPd62XsfU08plYm27naOUZO5avIPl1YO5I6 +Gi4kPdTvv3WFIy+QvoKoPhPCaD6EbdBpe8BbTQIDAQABAoIBABghJsCFfucKHdOE +RWZziHx22cblW6aML41wzTcLBFixdhx+lafCEwzF551OgZPbn5wwB4p0R3xAPAm9 +X0yEmnd8gEmtG+aavmg+rZ6sNbULhXenpvi4D4PR5uP61OX2rrEsvpgB0L9mYq0m +ah5VXvFdYzYcHDwTSsoMa+XgcbZ2qCW6Si3jnbBA1TPIJS5GjfPUQlu9g2FKQL5H +tlJ7L4Wq39zkueS6LH7kEXOoM+jHgA8F4f7MIrajmilYqnuXanVcMV3+K/6FvH2B +VBiLggG3CerhB3QyEvZBshvEvvcyRff2NK64CGr/xrAElj4cPHk/E499M1uvUXjE +boCrD+ECgYEA9LvLljf59h8WWF4bKQZGNKprgFdQIZ2iCEf+VGdGWt/mNg+LyXyn +3gS/vReON1eaMuEGklZM4Guh/ZPhsPaNmlu16PjmeYTIW1vQTHiO3KR7tAmWep70 +w+gVxDDzuRvBkuDF5oQsZnD3Ri9I7r+J5y9OhyZUsDXe/LJARivF3x0CgYEA2rRx +wl4mfuYmikvcO8I4vuKXcK1UyYmZQLhp6EHKfhSVgrt7XsstZX9AP2OxUUAocRks +e6vU/sKUSni7TQrZzAZHc8JXonDgmCqoMPBXIuUncvysGR1kmgVIbN8ISPKJuZoV +8Dbj3fQfHZ0g0R+mUcuZ+xBO5CKcjPWHZXZoxfECgYAQ/5o8bNbnyXD74k1wpAbs +UYn1+BqQuyot+RIpOqMgXLzYtGu5Kvdd7GaE88XlAiirsAWM1IGydMdjnYnniLh9 +KDGSZPddKWPhNJdbOGRz3tjYwHG7Qp8tnEkmv1+uU8c2NHaKdFPBKceDEHW4X4Vs +kVSa/oaTVqqOUrM0LIYp4QKBgQCW1aIriiGEnZhxAvbGJCJczAvkAzcZtBOFBmrM +ayuLnwiqXEEu1HPfr06RKWFuhxAdSF5cgNrqRSpe3jtXXCdvxdjbpmooNy8+4xSS +g/+kqmR1snvC6nmqnAAiTgP5w4RnBDUjMcggGLCpDOhIMkrT2Na+x7WRM6nCsceK +m4qREQKBgEWqdb/QkOMvvKAz2DPDeSrwlTyisrZu1G/86uE3ESb97DisPK+TF2Ts +r4RGUlKL79W3j5xjvIvqGEEDLC+8QKpay9OYXk3lbViPGB8akWMSP6Tw/8AedhVu +sjFqcBEFGOELwm7VjAcDeP6bXeXibFe+rysBrfFHUGllytCmNoAV +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_jwt_auth/jwks_server/server.py b/tests/integration/test_jwt_auth/jwks_server/server.py new file mode 100644 index 000000000000..96e07f02335e --- /dev/null +++ b/tests/integration/test_jwt_auth/jwks_server/server.py @@ -0,0 +1,33 @@ +import sys + +from bottle import response, route, run + + +@route("/.well-known/jwks.json") +def server(): + result = { + "keys": [ + { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": "0RRsKcZ5j9UckjioG4Phvav3dkg2sXP6tQ7ug0yowAo_u2HffB-1OjKuhWTpA3E3YkMKj0RrT-tuUpmZEXqCAipEV7XcfCv3o" + "7Poa7HTq1ti_abVwT_KyfGjoNBBSJH4LTNAyo2J8ySKSDtpAEU52iL7s40Ra6I0vqp7_aRuPF5M4zcHzN3zarG5EfSVSG1-gT" + "kaRv8XJbra0IeIINmKv0F4--ww8ZxXTR6cvI-MsArUiAPwzf7s5dMR4DNRG6YNTrPA0pTOqQE9sRPd62XsfU08plYm27naOUZ" + "O5avIPl1YO5I6Gi4kPdTvv3WFIy-QvoKoPhPCaD6EbdBpe8BbTQ", + "e": "AQAB"}, + ] + } + response.status = 200 + response.content_type = "application/json" + return result + + +@route("/") +def ping(): + response.content_type = "text/plain" + response.set_header("Content-Length", 2) + return "OK" + + +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_jwt_auth/test.py b/tests/integration/test_jwt_auth/test.py new file mode 100644 index 000000000000..6a1e1fe68e72 --- /dev/null +++ b/tests/integration/test_jwt_auth/test.py @@ -0,0 +1,101 @@ +import os +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_mock_servers + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "instance", + main_configs=["configs/validators.xml"], + user_configs=["configs/users.xml"], + with_minio=True, + # We actually don't need minio, but we need to run dummy resolver + # (a shortcut not to change cluster.py in a more unclear way, TBC later). +) +client = cluster.add_instance( + "client", +) + + +def run_jwks_server(): + script_dir = os.path.join(os.path.dirname(__file__), "jwks_server") + start_mock_servers( + cluster, + script_dir, + [ + ("server.py", "resolver", "8080"), + ], + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + run_jwks_server() + yield cluster + finally: + cluster.shutdown() + + +def curl_with_jwt(token, ip, https=False): + http_prefix = "https" if https else "http" + curl = f'curl -H "X-ClickHouse-JWT-Token: Bearer {token}" "{http_prefix}://{ip}:8123/?query=SELECT%20currentUser()"' + return curl + + +# See helpers/ directory if you need to re-create tokens (or understand how they are created) +def test_static_key(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJqd3RfdXNlciJ9." + "kfivQ8qD_oY0UvihydeadD7xvuiO3zSmhFOc_SGbEPQ", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" + + +def test_static_jwks(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Im15a2lkIn0." + "eyJzdWIiOiJqd3RfdXNlciIsImlzcyI6InRlc3RfaXNzIn0." + "CUioyRc_ms75YWkUwvPgLvaVk2Wmj8RzgqDALVd9LWUzCL5aU4yc_YaA3qnG_NoHd0uUF4FUjLxiocRoKNEgsE2jj7g_" + "wFMC5XHSHuFlfIZjovObXQEwGcKpXO2ser7ANu3k2jBC2FMpLfr_sZZ_GYSnqbp2WF6-l0uVQ0AHVwOy4x1Xkawiubkg" + "W2I2IosaEqT8QNuvvFWLWc1k-dgiNp8k6P-K4D4NBQub0rFlV0n7AEKNdV-_AEzaY_IqQT0sDeBSew_mdR0OH_N-6-" + "FmWWIroIn2DQ7pq93BkI7xdkqnxtt8RCWkCG8JLcoeJt8sHh7uTKi767loZJcPPNaxKA", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" + + +def test_jwks_server(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzUxMiIsImtpZCI6Im15a2lkIn0." + "eyJzdWIiOiJqd3RfdXNlciIsImlzcyI6InRlc3RfaXNzIn0.MjegqrrVyrMMpkxIM-J_q-" + "Sw68Vk5xZuFpxecLLMFs5qzvnh0jslWtyRfi-ANJeJTONPZM5m0yP1ITt8BExoHWobkkR11bXz0ylYEIOgwxqw" + "36XhL2GkE17p-wMvfhCPhGOVL3b7msDRUKXNN48aAJA-NxRbQFhMr-eEx3HsrZXy17Qc7z-" + "0dINe355kzAInGp6gMk3uksAlJ3vMODK8jE-WYFqXusr5GFhXubZXdE2mK0mIbMUGisOZhZLc4QVwvUsYDLBCgJ2RHr5vm" + "jp17j_ZArIedUJkjeC4o72ZMC97kLVnVw94QJwNvd4YisxL6A_mWLTRq9FqNLD4HmbcOQ", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" From 0dfe6997d1458c2b3f2d5a753ca5075328e7e35d Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 16 Jul 2025 23:13:11 +0000 Subject: [PATCH 02/24] add keykloak support(2) --- src/Access/AccessTokenProcessor.cpp | 115 +++++++++++++++++++++------- src/Access/AccessTokenProcessor.h | 18 +++-- src/Access/JWTValidator.h | 3 + 3 files changed, 103 insertions(+), 33 deletions(-) diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp index 9d6cc907afe6..86c2b1ad44f6 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/AccessTokenProcessor.cpp @@ -128,7 +128,7 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso if (is_auto && !is_manual) { - return std::make_unique(name, cache_lifetime, email_regex_str, config.getString(prefix + ".configuration_endpoint")); + return std::make_unique(name, cache_lifetime, email_regex_str, config.getString(prefix + ".configuration_endpoint"), config.getString(prefix + ".groups_claim_name", "")); } else if (!is_auto && is_manual) { @@ -335,57 +335,120 @@ String AzureAccessTokenProcessor::validateTokenAndGetUsername(const String & tok return getValueByKey(user_info_json, "sub"); } + +OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str, + const String & userinfo_endpoint_, + const String & token_introspection_endpoint_, + const String & jwks_uri_, + const String & groups_claim_name_) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), + userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_), groups_claim_name(groups_claim_name_) +{ + if (!jwks_uri_.empty()) + { + jwt_validator.emplace(name_ + "jwks_val", jwks_uri_, cache_invalidation_interval_); + } +} + OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, const UInt64 cache_invalidation_interval_, const String & email_regex_str, - const String & openid_config_endpoint_) + const String & openid_config_endpoint_, + const String & groups_claim_name_) : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) { const picojson::object openid_config = getObjectFromURI(Poco::URI(openid_config_endpoint_)); if (!openid_config.contains("userinfo_endpoint") || !openid_config.contains("introspection_endpoint")) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Cannot extract userinfo_endpoint or introspection_endpoint from OIDC configuration, consider manual configuration.", name); + + OpenIDAccessTokenProcessor(name_, + cache_invalidation_interval_, + email_regex_str, + getValueByKey(openid_config, "userinfo_endpoint"), + getValueByKey(openid_config, "introspection_endpoint"), + openid_config.contains("jwks_uri") ? getValueByKey(openid_config, "jwks_uri") : "", + groups_claim_name_); } bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) { const String & token = credentials.getToken(); + String username; + picojson::object user_info_json; - try + if (jwt_validator.has_value() && jwt_validator.value().validate("", token, username)) { - String username = validateTokenAndGetUsername(token); - if (!username.empty()) + + try { - /// Credentials are passed as const everywhere up the flow, so we have to comply, - /// in this case const_cast looks acceptable. - const_cast(credentials).setUserName(username); + auto decoded_token = jwt::decode(token); + user_info_json = decoded_token.get_payload_json(); + + /// TODO: Now we work only with Keycloak -- and it provides expires_at in token itself. Need to add actual token introspection logic for other OIDC providers. + if (decoded_token.has_expires_at()) + const_cast(credentials).setExpiresAt(decoded_token.get_expires_at()); } - else - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username with token", name); + catch (const std::exception & ex) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to process token as JWT: {}", name, ex.what()); + } + } + + /// If username or user info is empty -- local validation failed, trying introspection via provider + if (username.empty() || user_info_json.empty()) + { + try + { + user_info_json = getObjectFromURI(userinfo_endpoint, token); + username = getValueByKey(user_info_json, "sub"); + } + catch (...) + { + return false; + } + } + if (user_info_json.empty()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to obtain user info", name); + return false; } - catch (...) + else if (username.empty()) { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username", name); return false; } - return true; + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast is acceptable. + const_cast(credentials).setUserName(username); - /// TODO: add proper groups functionality -// try -// { -// const_cast(credentials).setExpiresAt(jwt::decode(token).get_expires_at()); -// } -// catch (...) { -// LOG_TRACE(getLogger("AccessTokenProcessor"), -// "{}: No expiration data found in a valid token, will use default cache lifetime", name); -// } -} + /// For now, list of groups is expected in a claim with specified name either in token itself or in userinfo response (Keycloak works this way) + /// TODO: add support for custom endpoints for retrieving groups. Keycloak lists groups in /userinfo and token itself, which is not always the case. + if (!groups_claim_name.empty() && user_info_json.contains(groups_claim_name)) + { + if (!user_info_json[groups_claim_name].is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to extract groups: invalid content in user data", name); + return true; + } -String OpenIDAccessTokenProcessor::validateTokenAndGetUsername(const String & token) const -{ - picojson::object user_info_json = getObjectFromURI(userinfo_endpoint, token); - return getValueByKey(user_info_json, "sub"); + std::set external_groups_names; + + picojson::array groups_array = user_info_json[groups_claim_name].get(); + for (const auto & group: groups_array) + { + if (group.is()) + external_groups_names.insert(group.get()); + } + const_cast(credentials).setGroups(external_groups_names); + } + + return true; } } diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h index 125b6d87b5c9..6f21f8b813be 100644 --- a/src/Access/AccessTokenProcessor.h +++ b/src/Access/AccessTokenProcessor.h @@ -106,24 +106,28 @@ class OpenIDAccessTokenProcessor : public IAccessTokenProcessor OpenIDAccessTokenProcessor(const String & name_, const UInt64 cache_invalidation_interval_, const String & email_regex_str, - const String & openid_config_endpoint_); + const String & openid_config_endpoint_, + const String & groups_claim_name_); /// Specify endpoints manually OpenIDAccessTokenProcessor(const String & name_, const UInt64 cache_invalidation_interval_, const String & email_regex_str, const String & userinfo_endpoint_, - const String & token_introspection_endpoint_) - : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), - userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_) {} + const String & token_introspection_endpoint_, + const String & jwks_uri_, + const String & groups_claim_name_); bool resolveAndValidate(const TokenCredentials & credentials) override; private: - const Poco::URI userinfo_endpoint; - const Poco::URI token_introspection_endpoint; + Poco::URI userinfo_endpoint; + Poco::URI token_introspection_endpoint; + /// Access token is often a valid JWT, so we can validate it locally to avoid unnecesary network requests. + std::optional jwt_validator = std::nullopt; - String validateTokenAndGetUsername(const String & token) const; + /// groups are expected under /userinfo endpoint under specified name + const String groups_claim_name; }; } diff --git a/src/Access/JWTValidator.h b/src/Access/JWTValidator.h index 03bcdf4990d1..2ab3f164d010 100644 --- a/src/Access/JWTValidator.h +++ b/src/Access/JWTValidator.h @@ -61,6 +61,9 @@ class JWKSValidator : public IJWTValidator public: explicit JWKSValidator(const String & name_, std::shared_ptr provider_) : IJWTValidator(name_), provider(provider_) {} + + explicit JWKSValidator(const String & name_, const String & uri, const size_t refresh_ms_) + : JWKSValidator(name_, std::make_shared(uri, refresh_ms_)) {} private: void validateImpl(const jwt::decoded_jwt & token) const override; From 7292b2449378c7204c1cb94786170fb37c0e540b Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Thu, 17 Jul 2025 09:35:47 +0000 Subject: [PATCH 03/24] fix build --- src/Access/AccessTokenProcessor.cpp | 1 + src/Access/TokenAccessStorage.cpp | 7 ++++--- src/Access/TokenAccessStorage.h | 21 ++++++++++++--------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp index 86c2b1ad44f6..9e0d606ea587 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/AccessTokenProcessor.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index baccf3489815..bec67c303a69 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -341,8 +341,9 @@ void TokenAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String std::optional TokenAccessStorage::authenticateImpl( const Credentials & credentials, const Poco::Net::IPAddress & address, - [[maybe_unused]] const ExternalAuthenticators & external_authenticators, - [[maybe_unused]] bool throw_if_user_not_exists, + const ExternalAuthenticators & external_authenticators, + const ClientInfo & /* client_info */, + bool throw_if_user_not_exists, bool /* allow_no_password */, bool /* allow_plaintext_password */) const { @@ -356,7 +357,7 @@ std::optional TokenAccessStorage::authenticateImpl( { // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. if (throw_if_user_not_exists) - throwNotFound(AccessEntityType::USER, credentials.getUserName()); + throwNotFound(AccessEntityType::USER, credentials.getUserName(), getStorageName()); else return {}; } diff --git a/src/Access/TokenAccessStorage.h b/src/Access/TokenAccessStorage.h index 0908f30ea519..34d2a7507988 100644 --- a/src/Access/TokenAccessStorage.h +++ b/src/Access/TokenAccessStorage.h @@ -62,18 +62,21 @@ class TokenAccessStorage : public IAccessStorage bool areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; + void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name); + void assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const; + void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const; + +protected: std::optional findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override; - std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, - [[maybe_unused]] const ExternalAuthenticators & external_authenticators, - [[maybe_unused]] bool throw_if_user_not_exists, - bool allow_no_password, bool allow_plaintext_password) const override; - - - void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name); - void assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const; - void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const; + std::optional authenticateImpl(const Credentials & credentials, + const Poco::Net::IPAddress & address, + const ExternalAuthenticators & external_authenticators, + const ClientInfo & client_info, + bool throw_if_user_not_exists, + bool allow_no_password, + bool allow_plaintext_password) const override; }; } From 6c853efa681ee7cc9cf539a51ea146cd133610f9 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Tue, 22 Jul 2025 14:02:11 +0000 Subject: [PATCH 04/24] shitty fix for a shitty problem --- src/Server/TCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index d321bee2a7b4..532a1e4eace4 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -44,6 +43,7 @@ #include #include #include +#include #include "Common/OpenTelemetryTraceContext.h" #include #include From b3c49190dd8942c11fa420f83c0601a9284f68b7 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 23 Jul 2025 22:45:17 +0000 Subject: [PATCH 05/24] small fixes --- .../operations/external-authenticators/tokens.md | 16 +++++++++------- src/Access/AccessTokenProcessor.cpp | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 6340e318237e..4a85b78edc06 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -93,16 +93,18 @@ All this implies that the SQL-driven [Access Control and Account Management](/do ```xml - - gogoogle - - - - + + + processor_name + + + + + ``` **Parameters** -- `server` — Name of one of processors defined in `access_token_processors` config section described above. This parameter is mandatory and cannot be empty. +- `processor` — Name of one of processors defined in `access_token_processors` config section described above. This parameter is mandatory and cannot be empty. - `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the IdP. diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp index 9e0d606ea587..223a041e419f 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/AccessTokenProcessor.cpp @@ -129,11 +129,21 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso if (is_auto && !is_manual) { - return std::make_unique(name, cache_lifetime, email_regex_str, config.getString(prefix + ".configuration_endpoint"), config.getString(prefix + ".groups_claim_name", "")); + return std::make_unique(name, + cache_lifetime, + email_regex_str, + config.getString(prefix + ".configuration_endpoint"), + config.getString(prefix + ".groups_claim_name", "")); } else if (!is_auto && is_manual) { - return std::make_unique(name, cache_lifetime, email_regex_str, config.getString(prefix + ".userinfo_endpoint"), config.getString(prefix + ".token_introspection_endpoint")); + return std::make_unique(name, + cache_lifetime, + email_regex_str, + config.getString(prefix + ".userinfo_endpoint"), + config.getString(prefix + ".token_introspection_endpoint"), + config.getString(prefix + ".jwks_uri"), + config.getString(prefix + ".groups_claim_name")); } throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse access token processor {}: " @@ -349,6 +359,7 @@ OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, { if (!jwks_uri_.empty()) { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: JWKS URI set, local JWT processing will be attempted", name); jwt_validator.emplace(name_ + "jwks_val", jwks_uri_, cache_invalidation_interval_); } } @@ -382,7 +393,6 @@ bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre if (jwt_validator.has_value() && jwt_validator.value().validate("", token, username)) { - try { auto decoded_token = jwt::decode(token); From c3071d1b7c35bf835658fa3a567e70dc47d5c261 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 25 Jul 2025 10:11:02 +0000 Subject: [PATCH 06/24] better docs, small fixes --- .../operations/external-authenticators/tokens.md | 15 ++++++++++++--- src/Access/AccessTokenProcessor.cpp | 3 ++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 4a85b78edc06..c009630a91f5 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -41,11 +41,20 @@ Different providers have different sets of parameters. **Parameters** -- `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure". +- `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure", "OpenID". - `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. - `email_filter` -- Regex for validation of user emails. Optional parameter, only for Google IdP. -- `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, only for Azure IdP. -- `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, only for Azure IdP. +- `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, used only for Azure IdP. +- `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, used only for Azure IdP. +- `groups_claim_name` -- Name of claim (field) that contains list of groups user belongs to. This claim will be looked up in the token itself (in case token is a valid JWT, e.g. in Keycloak) or in response from `/userinfo`. Optional parameter. +- `configuration_endpoint` -- URI of `.well-known/openid-configuration`. Optional parameter, useful only for OIDC-compliant providers (e.g. Keycloak). +- `userinfo_endpoint` -- URI of userinfo endpoint. Optional parameter. +- `token_introspection_endpoint` -- URI of token introspection endpoint. Optional parameter. + +:::note +Either `configuration_endpoint` or both `userinfo_endpoint` and `token_introspection_endpoint` shall be set. If none of them are set or all three are set, this is invalid configuration, it will not be parsed. +::: + ### Tokens cache To reduce number of requests to IdP, tokens are cached internally for no longer then `cache_lifetime` seconds. diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp index 223a041e419f..f6d7188666a7 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/AccessTokenProcessor.cpp @@ -92,6 +92,7 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso const String & prefix, const String & name) { + /// TODO: maybe bind external user to the processor it was created with? if (config.hasProperty(prefix + ".provider")) { String provider = Poco::toLower(config.getString(prefix + ".provider")); @@ -143,7 +144,7 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso config.getString(prefix + ".userinfo_endpoint"), config.getString(prefix + ".token_introspection_endpoint"), config.getString(prefix + ".jwks_uri"), - config.getString(prefix + ".groups_claim_name")); + config.getString(prefix + ".groups_claim_name", "")); } throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse access token processor {}: " From a91672d600f1b048cc873fddcf0540dfca5e9d6d Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 1 Aug 2025 16:04:35 +0000 Subject: [PATCH 07/24] pre-refactor tmp commit --- .../external-authenticators/tokens.md | 42 +++++++++++-------- src/Access/AccessControl.cpp | 7 ++++ src/Access/AccessTokenProcessor.cpp | 36 ++++++++-------- src/Access/AccessTokenProcessor.h | 7 ++-- src/Access/AuthenticationData.h | 4 ++ src/Access/ExternalAuthenticators.cpp | 37 ++++++++-------- src/Access/ExternalAuthenticators.h | 2 +- src/Access/TokenAccessStorage.cpp | 28 ++++++++----- src/Access/TokenAccessStorage.h | 4 +- 9 files changed, 97 insertions(+), 70 deletions(-) diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index c009630a91f5..1120d63e2000 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -13,25 +13,21 @@ OAuth 2.0 access tokens can be used to authenticate ClickHouse users. This works Though this authentication method is different from JWT authentication, it works under the same authentication method to maintain better compatibility. -For both of these approaches a definition of `access_token_processors` is mandatory. +For both of these approaches a definition of `token_processors` is mandatory. ## Access Token Processors -To define an access token processor, add `access_token_processors` section to `config.xml`. Example: +To define an access token processor, add `token_processors` section to `config.xml`. Example: ```xml - - - Google - ^[A-Za-z0-9._%+-]+@example\.com$ - 600 - + azure + claim_name CLIENT_ID TENANT_ID - + ``` @@ -42,11 +38,12 @@ Different providers have different sets of parameters. **Parameters** - `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure", "OpenID". +- `username_claim` -- name of claim (field) that will be treated as ClickHouse user name. Optional, default: "sub". - `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. - `email_filter` -- Regex for validation of user emails. Optional parameter, only for Google IdP. - `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, used only for Azure IdP. - `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, used only for Azure IdP. -- `groups_claim_name` -- Name of claim (field) that contains list of groups user belongs to. This claim will be looked up in the token itself (in case token is a valid JWT, e.g. in Keycloak) or in response from `/userinfo`. Optional parameter. +- `groups_claim` -- Name of claim (field) that contains list of groups user belongs to. This claim will be looked up in the token itself (in case token is a valid JWT, e.g. in Keycloak) or in response from `/userinfo`. Optional parameter. - `configuration_endpoint` -- URI of `.well-known/openid-configuration`. Optional parameter, useful only for OIDC-compliant providers (e.g. Keycloak). - `userinfo_endpoint` -- URI of userinfo endpoint. Optional parameter. - `token_introspection_endpoint` -- URI of token introspection endpoint. Optional parameter. @@ -67,17 +64,20 @@ Locally defined users can be authenticated with an access token. To allow this, ```xml - - - + + + + ``` +Inside `jwt` one or more specific access token processors names can be specified -- only those processors will be tried when authenticating. If no processors are specified, _all_ processors will be tried. + At each login attempt, ClickHouse will attempt to validate token and get user info against every defined access token provider. When SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled, users that are authenticated with tokens can also be created using the [CREATE USER](/docs/en/sql-reference/statements/create/user.md#create-user-statement) statement. @@ -94,7 +94,7 @@ If there is no suitable user pre-defined in ClickHouse, authentication is still To allow this, add `token` section to the `users_directories` section of the `config.xml` file. At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. -If the user is not defined, ClickHouse will treat user as externally defined, and will try to validate the token and get user information from the specified processor. +If the user is not defined, ClickHouse will treat the user as externally defined and will try to validate the token and get user information from the specified processor. If validated successfully, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. All this implies that the SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled and roles are created using the [CREATE ROLE](/docs/en/sql-reference/statements/create/role.md#create-role-statement) statement. @@ -105,15 +105,21 @@ All this implies that the SQL-driven [Access Control and Account Management](/do processor_name - + - + + ``` +:::note +For now, no more than one `token` section can be defined inside `user_directories`. This _may_ change in future. +::: + **Parameters** -- `processor` — Name of one of processors defined in `access_token_processors` config section described above. This parameter is mandatory and cannot be empty. -- `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the IdP. +- `processor` — Name of one of processors defined in `token_processors` config section described above. This parameter is mandatory and cannot be empty. +- `common_roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the IdP. Optional. +- `roles_filter` — Regex string for groups filtering. Only groups matching this regex will be mapped to roles. Optional. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 0cd64717b60a..a328e7df500d 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -42,6 +42,7 @@ namespace ErrorCodes extern const int REQUIRED_PASSWORD; extern const int CANNOT_COMPILE_REGEXP; extern const int BAD_ARGUMENTS; + extern const int INVALID_CONFIG_PARAMETER; } namespace @@ -440,6 +441,8 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( Strings keys_in_user_directories; config.keys(key, keys_in_user_directories); + bool has_token_storage = false; + for (const String & key_in_user_directories : keys_in_user_directories) { String prefix = key + "." + key_in_user_directories; @@ -490,7 +493,11 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( } else if (type == TokenAccessStorage::STORAGE_TYPE) { + if (has_token_storage) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Only one `token` section can be defined."); + addTokenStorage(name, config, prefix); + has_token_storage = true; } else throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown storage type '{}' at {} in config", type, prefix); diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp index f6d7188666a7..b6cab369f76d 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/AccessTokenProcessor.cpp @@ -97,11 +97,9 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso { String provider = Poco::toLower(config.getString(prefix + ".provider")); - String email_regex_str = config.hasProperty(prefix + ".email_filter") ? config.getString( - prefix + ".email_filter") : ""; - - UInt64 cache_lifetime = config.hasProperty(prefix + ".cache_lifetime") ? config.getUInt64( - prefix + ".cache_lifetime") : 3600; + String email_regex_str = config.getString(prefix + ".email_filter", ""); + UInt64 cache_lifetime = config.getUInt64(prefix + ".cache_lifetime", 3600); + String username_claim = config.getString(prefix + ".username_claim", "sub"); if (provider == "google") { @@ -134,7 +132,7 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso cache_lifetime, email_regex_str, config.getString(prefix + ".configuration_endpoint"), - config.getString(prefix + ".groups_claim_name", "")); + config.getString(prefix + ".groups_claim", "")); } else if (!is_auto && is_manual) { @@ -144,7 +142,7 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso config.getString(prefix + ".userinfo_endpoint"), config.getString(prefix + ".token_introspection_endpoint"), config.getString(prefix + ".jwks_uri"), - config.getString(prefix + ".groups_claim_name", "")); + config.getString(prefix + ".groups_claim", "")); } throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse access token processor {}: " @@ -165,7 +163,7 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre const String & token = credentials.getToken(); auto user_info = getUserInfo(token); - String user_name = user_info["sub"]; + String user_name = user_info[username_claim]; bool has_email = user_info.contains("email"); if (email_regex.ok()) @@ -249,7 +247,7 @@ std::unordered_map GoogleAccessTokenProcessor::getUserInfo(const try { user_info_map["email"] = getValueByKey(user_info_json, "email"); - user_info_map["sub"] = getValueByKey(user_info_json, "sub"); + user_info_map[username_claim] = getValueByKey(user_info_json, username_claim); return user_info_map; } catch (std::runtime_error & e) @@ -265,7 +263,7 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred /// We will not trust user data in this token except for 'exp' value to determine caching duration. /// Explanation here: https://stackoverflow.com/questions/60778634/failing-signature-validation-of-jwt-tokens-from-azure-ad /// Let Azure validate it: only valid tokens will be accepted. - /// Use GET https://graph.microsoft.com/oidc/userinfo to verify token and get sub at the same time + /// Use GET https://graph.microsoft.com/oidc/userinfo to verify token and get user info at the same time const String & token = credentials.getToken(); @@ -344,7 +342,7 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred String AzureAccessTokenProcessor::validateTokenAndGetUsername(const String & token) const { picojson::object user_info_json = getObjectFromURI(user_info_uri, token); - return getValueByKey(user_info_json, "sub"); + return getValueByKey(user_info_json, username_claim); } @@ -354,9 +352,9 @@ OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, const String & userinfo_endpoint_, const String & token_introspection_endpoint_, const String & jwks_uri_, - const String & groups_claim_name_) + const String & groups_claim_) : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), - userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_), groups_claim_name(groups_claim_name_) + userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_), groups_claim(groups_claim_) { if (!jwks_uri_.empty()) { @@ -369,7 +367,7 @@ OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, const UInt64 cache_invalidation_interval_, const String & email_regex_str, const String & openid_config_endpoint_, - const String & groups_claim_name_) + const String & groups_claim_) : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) { const picojson::object openid_config = getObjectFromURI(Poco::URI(openid_config_endpoint_)); @@ -383,7 +381,7 @@ OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, getValueByKey(openid_config, "userinfo_endpoint"), getValueByKey(openid_config, "introspection_endpoint"), openid_config.contains("jwks_uri") ? getValueByKey(openid_config, "jwks_uri") : "", - groups_claim_name_); + groups_claim_); } bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) @@ -415,7 +413,7 @@ bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre try { user_info_json = getObjectFromURI(userinfo_endpoint, token); - username = getValueByKey(user_info_json, "sub"); + username = getValueByKey(user_info_json, username_claim); } catch (...) { @@ -440,9 +438,9 @@ bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre /// For now, list of groups is expected in a claim with specified name either in token itself or in userinfo response (Keycloak works this way) /// TODO: add support for custom endpoints for retrieving groups. Keycloak lists groups in /userinfo and token itself, which is not always the case. - if (!groups_claim_name.empty() && user_info_json.contains(groups_claim_name)) + if (!groups_claim.empty() && user_info_json.contains(groups_claim)) { - if (!user_info_json[groups_claim_name].is()) + if (!user_info_json[groups_claim].is()) { LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to extract groups: invalid content in user data", name); @@ -451,7 +449,7 @@ bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre std::set external_groups_names; - picojson::array groups_array = user_info_json[groups_claim_name].get(); + picojson::array groups_array = user_info_json[groups_claim].get(); for (const auto & group: groups_array) { if (group.is()) diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h index 6f21f8b813be..46e64f35b53a 100644 --- a/src/Access/AccessTokenProcessor.h +++ b/src/Access/AccessTokenProcessor.h @@ -55,6 +55,7 @@ class IAccessTokenProcessor protected: const String name; + const String username_claim = "sub"; const UInt64 cache_invalidation_interval; re2::RE2 email_regex; @@ -107,7 +108,7 @@ class OpenIDAccessTokenProcessor : public IAccessTokenProcessor const UInt64 cache_invalidation_interval_, const String & email_regex_str, const String & openid_config_endpoint_, - const String & groups_claim_name_); + const String & groups_claim_); /// Specify endpoints manually OpenIDAccessTokenProcessor(const String & name_, @@ -116,7 +117,7 @@ class OpenIDAccessTokenProcessor : public IAccessTokenProcessor const String & userinfo_endpoint_, const String & token_introspection_endpoint_, const String & jwks_uri_, - const String & groups_claim_name_); + const String & groups_claim_); bool resolveAndValidate(const TokenCredentials & credentials) override; private: @@ -127,7 +128,7 @@ class OpenIDAccessTokenProcessor : public IAccessTokenProcessor std::optional jwt_validator = std::nullopt; /// groups are expected under /userinfo endpoint under specified name - const String groups_claim_name; + const String groups_claim; }; } diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h index 6bfeae4cea3b..bcd7bd61270f 100644 --- a/src/Access/AuthenticationData.h +++ b/src/Access/AuthenticationData.h @@ -82,6 +82,9 @@ class AuthenticationData const String & getJWTClaims() const { return jwt_claims; } void setJWTClaims(const String & jwt_claims_) { jwt_claims = jwt_claims_; } + const String & getTokenProcessorName() const { return token_processor_name; } + void setTokenProcessorName(const String & token_processor_name_) { token_processor_name = token_processor_name_; } + friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs); friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); } @@ -121,6 +124,7 @@ class AuthenticationData HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC; time_t valid_until = 0; String jwt_claims; + String token_processor_name; }; } diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 00ec7a7a5f5c..991f881a4deb 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -325,21 +325,22 @@ void parseJWTValidators(std::unordered_map> & access_token_processors, +void parseAccessTokenProcessors(std::unordered_map> & token_processors, const Poco::Util::AbstractConfiguration & config, - const String & access_token_processors_config, + const String & token_processors_config, LoggerPtr log) { - Poco::Util::AbstractConfiguration::Keys access_token_processors_keys; - config.keys(access_token_processors_config, access_token_processors_keys); - access_token_processors.clear(); + Poco::Util::AbstractConfiguration::Keys token_processors_keys; + config.keys(token_processors_config, token_processors_keys); - for (const auto & processor : access_token_processors_keys) + token_processors.clear(); + + for (const auto & processor : token_processors_keys) { - String prefix = fmt::format("{}.{}", access_token_processors_config, processor); + String prefix = fmt::format("{}.{}", token_processors_config, processor); try { - access_token_processors[processor] = IAccessTokenProcessor::parseTokenProcessor(config, prefix, processor); + token_processors[processor] = IAccessTokenProcessor::parseTokenProcessor(config, prefix, processor); } catch (...) { @@ -360,11 +361,11 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur std::size_t kerberos_keys_count = 0; std::size_t http_auth_server_keys_count = 0; std::size_t jwt_validators_count = 0; - std::size_t access_token_processors_count = 0; + std::size_t token_processors_count = 0; const String http_auth_servers_config = "http_authentication_servers"; const String jwt_validators_config = "jwt_validators"; - const String access_token_processors_config = "access_token_processors"; + const String token_processors_config = "token_processors"; for (auto key : all_keys) { @@ -378,7 +379,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur kerberos_keys_count += (key == "kerberos"); http_auth_server_keys_count += (key == http_auth_servers_config); jwt_validators_count += (key == jwt_validators_config); - access_token_processors_count += (key == access_token_processors_config); + token_processors_count += (key == token_processors_config); } if (ldap_servers_key_count > 1) @@ -393,8 +394,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur if (jwt_validators_count > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", jwt_validators_config); - if (access_token_processors_count > 1) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", access_token_processors_config); + if (token_processors_count > 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", token_processors_config); Poco::Util::AbstractConfiguration::Keys http_auth_server_names; config.keys(http_auth_servers_config, http_auth_server_names); @@ -452,7 +453,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur } parseJWTValidators(jwt_validators, config, jwt_validators_config, log); - parseAccessTokenProcessors(access_token_processors, config, access_token_processors_config, log); + parseAccessTokenProcessors(token_processors, config, token_processors_config, log); } static UInt128 computeParamsHash(const LDAPClient::Params & params, const LDAPClient::RoleSearchParamsList * role_search_params) @@ -685,7 +686,7 @@ bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials { std::lock_guard lock{mutex}; - if (access_token_processors.empty()) + if (token_processors.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); /// lookup token in local cache if not expired. @@ -707,7 +708,7 @@ bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials } } - for (const auto & it : access_token_processors) + for (const auto & it : token_processors) { if (it.second->resolveAndValidate(credentials)) { @@ -742,10 +743,10 @@ bool ExternalAuthenticators::checkAccessTokenCredentialsByExactProcessor(const T { std::lock_guard lock{mutex}; - if (access_token_processors.empty()) + if (token_processors.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); - for (const auto & it : access_token_processors) + for (const auto & it : token_processors) { if (name == it.second->getName()) { diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 311765560341..253c35bf5738 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -89,7 +89,7 @@ class ExternalAuthenticators std::optional kerberos_params TSA_GUARDED_BY(mutex) ; std::unordered_map http_auth_servers TSA_GUARDED_BY(mutex) ; std::unordered_map> jwt_validators TSA_GUARDED_BY(mutex) ; - std::unordered_map> access_token_processors TSA_GUARDED_BY(mutex) ; + std::unordered_map> token_processors TSA_GUARDED_BY(mutex) ; void resetImpl() TSA_REQUIRES(mutex); }; diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index bec67c303a69..388e3f13c0b5 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -20,12 +20,8 @@ namespace ErrorCodes TokenAccessStorage::TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) : IAccessStorage(storage_name_), access_control(access_control_), config(config_), prefix(prefix_), + roles_filter(config.getString(prefix.empty() ? "" : prefix + "." + "roles_filter", "")), memory_storage(storage_name_, access_control.getChangesNotifier(), false) -{ - setConfiguration(); -} - -void TokenAccessStorage::setConfiguration() { std::lock_guard lock(mutex); @@ -35,17 +31,14 @@ void TokenAccessStorage::setConfiguration() if (provider_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "'processor' must be specified for Token user directory"); - const bool has_roles = config.has(prefix_str + "roles"); - std::set common_roles_cfg; - if (has_roles) + if (config.has(prefix_str + "common_roles")) { Poco::Util::AbstractConfiguration::Keys role_names; config.keys(prefix_str + "roles", role_names); common_roles_cfg.insert(role_names.begin(), role_names.end()); } - common_role_names.swap(common_roles_cfg); external_role_hashes.clear(); @@ -375,7 +368,22 @@ std::optional TokenAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); - std::set external_roles = token_credentials.getGroups(); + std::set external_roles; + if (!roles_filter.ok()) + { + external_roles = token_credentials.getGroups(); + LOG_TRACE(getLogger(), "{}: No external role filtering set, applying all available groups", getStorageName()); + } + else + { + for (const auto & group: token_credentials.getGroups()) { + if (RE2::FullMatch(group, roles_filter)) + { + external_roles.insert(group); + LOG_TRACE(getLogger(), "{}: Granted role (group) {} to user", getStorageName(), user->getName()); + } + } + } if (new_user) { diff --git a/src/Access/TokenAccessStorage.h b/src/Access/TokenAccessStorage.h index 34d2a7507988..d9fdf758d25f 100644 --- a/src/Access/TokenAccessStorage.h +++ b/src/Access/TokenAccessStorage.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -47,6 +48,7 @@ class TokenAccessStorage : public IAccessStorage const String & prefix; String provider_name; + re2::RE2 roles_filter; std::set common_role_names; // role name that should be granted to all users at all times mutable std::map external_role_hashes; @@ -57,7 +59,7 @@ class TokenAccessStorage : public IAccessStorage scope_guard role_change_subscription; mutable MemoryAccessStorage memory_storage; - void setConfiguration(); +// void setConfiguration(); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); bool areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; From 5282080df81f05677cb969a873d28c8d2239beb0 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Sun, 3 Aug 2025 18:57:44 +0000 Subject: [PATCH 08/24] major refactor --- .../operations/external-authenticators/jwt.md | 43 +- .../external-authenticators/tokens.md | 7 +- src/Access/AccessControl.cpp | 5 - src/Access/AccessControl.h | 2 - src/Access/AccessTokenProcessor.h | 134 ------ src/Access/Authentication.cpp | 5 +- src/Access/ExternalAuthenticators.cpp | 174 +++----- src/Access/ExternalAuthenticators.h | 41 +- src/Access/JWTValidator.cpp | 367 ----------------- src/Access/JWTValidator.h | 72 ---- src/Access/TokenAccessStorage.cpp | 4 +- src/Access/TokenProcessors.h | 166 ++++++++ src/Access/TokenProcessorsJWT.cpp | 386 ++++++++++++++++++ ...rocessor.cpp => TokenProcessorsOpaque.cpp} | 257 ++++-------- src/Access/TokenProcessorsParse.cpp | 100 +++++ src/Server/HTTP/authenticateUserByHTTP.cpp | 3 +- src/Server/TCPHandler.cpp | 3 +- 17 files changed, 811 insertions(+), 958 deletions(-) delete mode 100644 src/Access/AccessTokenProcessor.h delete mode 100644 src/Access/JWTValidator.cpp delete mode 100644 src/Access/JWTValidator.h create mode 100644 src/Access/TokenProcessors.h create mode 100644 src/Access/TokenProcessorsJWT.cpp rename src/Access/{AccessTokenProcessor.cpp => TokenProcessorsOpaque.cpp} (52%) create mode 100644 src/Access/TokenProcessorsParse.cpp diff --git a/docs/en/operations/external-authenticators/jwt.md b/docs/en/operations/external-authenticators/jwt.md index fbc36f8399f7..4b44239f8f37 100644 --- a/docs/en/operations/external-authenticators/jwt.md +++ b/docs/en/operations/external-authenticators/jwt.md @@ -94,14 +94,8 @@ Only one of `static_jwks` or `static_jwks_file` keys must be present in one veri - http://localhost:8000/.well-known/jwks.json - 1000 - 1000 - 1000 - 3 - 50 - 1000 - 300000 + http://localhost:8000/.well-known/jwks.json + 300000 @@ -110,38 +104,7 @@ Only one of `static_jwks` or `static_jwks_file` keys must be present in one veri #### Parameters: - `uri` - JWKS endpoint. Mandatory. -- `refresh_ms` - Period for resend request for refreshing JWKS. Optional, default: 300000. - -Timeouts in milliseconds on the socket used for communicating with the server (optional): -- `connection_timeout_ms` - Default: 1000. -- `receive_timeout_ms` - Default: 1000. -- `send_timeout_ms` - Default: 1000. - -Retry parameters (optional): -- `max_tries` - The maximum number of attempts to make an authentication request. Default: 3. -- `retry_initial_backoff_ms` - The backoff initial interval on retry. Default: 50. -- `retry_max_backoff_ms` - The maximum backoff interval. Default: 1000. - -### Verifying access tokens {$verifying-access-tokens} - -Access tokens that are not JWT (and thus no data can be extracted from the token directly) need to be resolved by external providers. - -**Example** -```xml - - - - - google - - - -``` - -#### Parameters: - -- `provider` - name of provider that will be used for token processing. Mandatory parameter. Possible options: `google`. - +- `jwks_refresh_timeout` - Period for resend request for refreshing JWKS. Optional, default: 300000. ### Enabling JWT authentication in `users.xml` {#enabling-jwt-auth-in-users-xml} diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 1120d63e2000..7a2a2683bcd2 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -24,8 +24,6 @@ To define an access token processor, add `token_processors` section to `config.x azure claim_name - CLIENT_ID - TENANT_ID @@ -39,10 +37,7 @@ Different providers have different sets of parameters. - `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure", "OpenID". - `username_claim` -- name of claim (field) that will be treated as ClickHouse user name. Optional, default: "sub". -- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. -- `email_filter` -- Regex for validation of user emails. Optional parameter, only for Google IdP. -- `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, used only for Azure IdP. -- `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, used only for Azure IdP. +- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. - `groups_claim` -- Name of claim (field) that contains list of groups user belongs to. This claim will be looked up in the token itself (in case token is a valid JWT, e.g. in Keycloak) or in response from `/userinfo`. Optional parameter. - `configuration_endpoint` -- URI of `.well-known/openid-configuration`. Optional parameter, useful only for OIDC-compliant providers (e.g. Keycloak). - `userinfo_endpoint` -- URI of userinfo endpoint. Optional parameter. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index a328e7df500d..30ad7999c8ff 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -728,11 +728,6 @@ bool AccessControl::isNoPasswordAllowed() const return allow_no_password; } -bool AccessControl::isJWTEnabled() const -{ - return external_authenticators->isJWTAllowed(); -} - void AccessControl::setPlaintextPasswordAllowed(bool allow_plaintext_password_) { allow_plaintext_password = allow_plaintext_password_; diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index 76c0d63c216f..f8ff12730088 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -157,8 +157,6 @@ class AccessControl : public MultipleAccessStorage void setNoPasswordAllowed(bool allow_no_password_); bool isNoPasswordAllowed() const; - bool isJWTEnabled() const; - /// Allows users with plaintext password (by default it's allowed). void setPlaintextPasswordAllowed(bool allow_plaintext_password_); bool isPlaintextPasswordAllowed() const; diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h deleted file mode 100644 index 46e64f35b53a..000000000000 --- a/src/Access/AccessTokenProcessor.h +++ /dev/null @@ -1,134 +0,0 @@ -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int AUTHENTICATION_FAILED; - extern const int INVALID_CONFIG_PARAMETER; -} - -//class GoogleAccessTokenProcessor; - -class IAccessTokenProcessor -{ -public: - IAccessTokenProcessor(const String & name_, - const UInt64 cache_invalidation_interval_, - const String & email_regex_str) - : name(name_), - cache_invalidation_interval(cache_invalidation_interval_), - email_regex(email_regex_str) - { - if (!email_regex_str.empty()) - { - /// Later, we will use .ok() to determine whether there was a regex specified in config or not. - if (!email_regex.ok()) - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Invalid regex in definition of access token processor {}", name); - } - } - - virtual ~IAccessTokenProcessor() = default; - - String getName() { return name; } - UInt64 getCacheInvalidationInterval() { return cache_invalidation_interval; } - - virtual bool resolveAndValidate(const TokenCredentials & credentials) = 0; - - static std::unique_ptr parseTokenProcessor( - const Poco::Util::AbstractConfiguration & config, - const String & prefix, - const String & name); - -protected: - const String name; - const String username_claim = "sub"; - const UInt64 cache_invalidation_interval; - re2::RE2 email_regex; - - bool valid; -}; - - -class GoogleAccessTokenProcessor : public IAccessTokenProcessor -{ -public: - GoogleAccessTokenProcessor(const String & name_, - const UInt64 cache_invalidation_interval_, - const String & email_regex_str) - : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) {} - - bool resolveAndValidate(const TokenCredentials & credentials) override; - -private: - [[maybe_unused]] static const Poco::URI token_info_uri; - static const Poco::URI user_info_uri; - - std::unordered_map getUserInfo(const String & token) const; -}; - - -class AzureAccessTokenProcessor : public IAccessTokenProcessor -{ -public: - AzureAccessTokenProcessor(const String & name_, - const UInt64 cache_invalidation_interval_, - const String & email_regex_str, - const String & tenant_id_) - : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), - jwks_uri_str("https://login.microsoftonline.com/" + tenant_id_ + "/discovery/v2.0/keys") {} - - bool resolveAndValidate(const TokenCredentials & credentials) override; -private: - static const Poco::URI user_info_uri; - - const String jwks_uri_str; - - String validateTokenAndGetUsername(const String & token) const; -}; - -class OpenIDAccessTokenProcessor : public IAccessTokenProcessor -{ -public: - /// Obtain endpoints from openid-configuration URL - OpenIDAccessTokenProcessor(const String & name_, - const UInt64 cache_invalidation_interval_, - const String & email_regex_str, - const String & openid_config_endpoint_, - const String & groups_claim_); - - /// Specify endpoints manually - OpenIDAccessTokenProcessor(const String & name_, - const UInt64 cache_invalidation_interval_, - const String & email_regex_str, - const String & userinfo_endpoint_, - const String & token_introspection_endpoint_, - const String & jwks_uri_, - const String & groups_claim_); - - bool resolveAndValidate(const TokenCredentials & credentials) override; -private: - Poco::URI userinfo_endpoint; - Poco::URI token_introspection_endpoint; - - /// Access token is often a valid JWT, so we can validate it locally to avoid unnecesary network requests. - std::optional jwt_validator = std::nullopt; - - /// groups are expected under /userinfo endpoint under specified name - const String groups_claim; -}; - -} diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index a7ae9022ec6b..19fddbfb8421 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -345,10 +345,7 @@ bool Authentication::areCredentialsValid( if (authentication_method.getType() != AuthenticationType::JWT) return false; - if (external_authenticators.checkJWTClaims(authentication_method.getJWTClaims(), *token_credentials)) - return true; - - return external_authenticators.checkAccessTokenCredentials(*token_credentials); + return external_authenticators.checkTokenCredentials(*token_credentials); } if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 991f881a4deb..a05ce5874340 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -287,13 +286,7 @@ void ExternalAuthenticators::resetImpl() ldap_client_params_blueprint.clear(); ldap_caches.clear(); kerberos_params.reset(); - jwt_validators.clear(); -} - -bool ExternalAuthenticators::isJWTAllowed() const -{ - std::lock_guard lock(mutex); - return !jwt_validators.empty(); + token_processors.clear(); } void ExternalAuthenticators::reset() @@ -302,30 +295,7 @@ void ExternalAuthenticators::reset() resetImpl(); } -void parseJWTValidators(std::unordered_map> & jwt_validators, - const Poco::Util::AbstractConfiguration & config, - const String & jwt_validators_config, - LoggerPtr log) -{ - Poco::Util::AbstractConfiguration::Keys jwt_validators_keys; - config.keys(jwt_validators_config, jwt_validators_keys); - jwt_validators.clear(); - for (const auto & jwt_validator : jwt_validators_keys) - { - if (jwt_validator == "settings_key") continue; - String prefix = fmt::format("{}.{}", jwt_validators_config, jwt_validator); - try - { - jwt_validators[jwt_validator] = IJWTValidator::parseJWTValidator(config, prefix, jwt_validator); - } - catch (...) - { - tryLogCurrentException(log, "Could not parse JWT validator" + backQuote(jwt_validator)); - } - } -} - -void parseAccessTokenProcessors(std::unordered_map> & token_processors, +void parseTokenProcessors(std::unordered_map> & token_processors, const Poco::Util::AbstractConfiguration & config, const String & token_processors_config, LoggerPtr log) @@ -340,11 +310,11 @@ void parseAccessTokenProcessors(std::unordered_mapsecond; } -bool ExternalAuthenticators::resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured = true) const +bool ExternalAuthenticators::checkCredentialsAgainstProcessor(const ITokenProcessor & processor, + const TokenCredentials & credentials) const { - std::lock_guard lock{mutex}; - - const auto token = String(credentials.getToken()); + if (processor.resolveAndValidate(credentials)) + { + TokenCacheEntry cache_entry; + cache_entry.user_name = credentials.getUserName(); + cache_entry.external_roles = credentials.getGroups(); - if (jwt_validators.empty() && throw_not_configured) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + auto default_expiration_ts = std::chrono::system_clock::now() + + std::chrono::minutes(processor.getTokenCacheLifetime()); - for (const auto & it : jwt_validators) - { - String username; - if (it.second->validate("", token, username)) + if (credentials.getExpiresAt().has_value()) { - /// Credentials are passed as const everywhere up the flow, so we have to comply, - /// in this case const_cast looks acceptable. - const_cast(credentials).setUserName(username); - LOG_TRACE(getLogger("JWTAuthentication"), "Extracted username {} from JWT by {}", username, it.first); - return true; + if (credentials.getExpiresAt().value() < default_expiration_ts) + cache_entry.expires_at = credentials.getExpiresAt().value(); + } + else + { + cache_entry.expires_at = default_expiration_ts; } - LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); - } - return false; -} - -bool ExternalAuthenticators::checkJWTClaims(const String & claims, const TokenCredentials & credentials) const -{ - std::lock_guard lock{mutex}; - - const auto token = String(credentials.getToken()); - if (jwt_validators.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), processor.getProcessorName()); - for (const auto & it : jwt_validators) - { - String username; - if (it.second->validate(claims, token, username)) + // CHeck if a cache entry for the same user but with another token exists -- old cache entry is considered outdated and removed + auto old_token_iter = username_to_access_token_cache.find(cache_entry.user_name); + if (old_token_iter != username_to_access_token_cache.end()) { - /// Credentials are passed as const everywhere up the flow, so we have to comply, - /// in this case const_cast looks acceptable. - const_cast(credentials).setUserName(username); - LOG_DEBUG(getLogger("JWTAuthentication"), "Authenticated with JWT for {} by {}", username, it.first); - return true; + access_token_to_username_cache.erase(old_token_iter->second); + username_to_access_token_cache.erase(old_token_iter); } - LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); + + access_token_to_username_cache[credentials.getToken()] = cache_entry; + username_to_access_token_cache[cache_entry.user_name] = credentials.getToken(); + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} added", cache_entry.user_name); + + return true; } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by {}", processor.getProcessorName()); + return false; } -bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials & credentials) const +bool ExternalAuthenticators::checkTokenCredentials(const TokenCredentials & credentials, const String & processor_name) const { std::lock_guard lock{mutex}; @@ -690,13 +652,14 @@ bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); /// lookup token in local cache if not expired. - auto cached_entry_iter = access_token_cache.find(credentials.getToken()); - if (cached_entry_iter != access_token_cache.end()) + auto cached_entry_iter = access_token_to_username_cache.find(credentials.getToken()); + if (cached_entry_iter != access_token_to_username_cache.end()) { - if (cached_entry_iter->second.expires_at <= std::chrono::system_clock::now()) + if (cached_entry_iter->second.expires_at <= std::chrono::system_clock::now()) // Token found in cache, but already outdated -- need to remove it. { LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} expired, removing", cached_entry_iter->second.user_name); - access_token_cache.erase(cached_entry_iter); + access_token_to_username_cache.erase(cached_entry_iter); + username_to_access_token_cache.erase(cached_entry_iter->second.user_name); } else { @@ -708,60 +671,17 @@ bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials } } - for (const auto & it : token_processors) - { - if (it.second->resolveAndValidate(credentials)) - { - AccessTokenCacheEntry cache_entry; - cache_entry.user_name = credentials.getUserName(); - cache_entry.external_roles = credentials.getGroups(); - - auto default_expiration_ts = std::chrono::system_clock::now() - + std::chrono::minutes(it.second->getCacheInvalidationInterval()); - - if (credentials.getExpiresAt().has_value()) - { - if (credentials.getExpiresAt().value() < default_expiration_ts) - cache_entry.expires_at = credentials.getExpiresAt().value(); - } - else - { - cache_entry.expires_at = default_expiration_ts; - } - LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} added", cache_entry.user_name); - - access_token_cache[credentials.getToken()] = cache_entry; - LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), it.first); - return true; - } - LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by {}", it.first); - } - return false; -} - -bool ExternalAuthenticators::checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const -{ - std::lock_guard lock{mutex}; - - if (token_processors.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); - - for (const auto & it : token_processors) + if (processor_name.empty()) { - if (name == it.second->getName()) + for (const auto & it: token_processors) { - if (it.second->resolveAndValidate(credentials)) { - LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", - credentials.getUserName(), it.first); + if (checkCredentialsAgainstProcessor(*it.second, credentials)) return true; - } else - { - LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by processor {}", name); - return false; - } } } - LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token: no processor with name {}", name); + else + return token_processors.contains(processor_name) && checkCredentialsAgainstProcessor(*token_processors[processor_name], credentials); + return false; } diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 253c35bf5738..d5cc36098a12 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -1,10 +1,9 @@ #pragma once -#include +#include #include #include #include -#include #include #include #include @@ -49,16 +48,10 @@ class ExternalAuthenticators bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; bool checkHTTPBasicCredentials(const String & server, const BasicCredentials & credentials, const ClientInfo & client_info, SettingsChanges & settings) const; - bool resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured) const; - bool checkJWTClaims(const String & claims, const TokenCredentials & credentials) const; - - bool checkAccessTokenCredentials(const TokenCredentials & credentials) const; - bool checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const; + bool checkTokenCredentials(const TokenCredentials & credentials, const String & processor_name = "") const; GSSAcceptorContext::Params getKerberosParams() const; - bool isJWTAllowed() const; - private: HTTPAuthClientParams getHTTPAuthenticationParams(const String& server) const; @@ -69,27 +62,33 @@ class ExternalAuthenticators LDAPClient::SearchResultsList last_successful_role_search_results; }; - struct AccessTokenCacheEntry - { - std::chrono::system_clock::time_point expires_at; - String user_name; - std::set external_roles; - }; - using LDAPCache = std::unordered_map; // user name -> cache entry using LDAPCaches = std::map; // server name -> cache using LDAPParams = std::map; // server name -> params - using AccessTokenCache = std::unordered_map; // Access token -> cache entry - mutable std::mutex mutex; LDAPParams ldap_client_params_blueprint TSA_GUARDED_BY(mutex) ; mutable LDAPCaches ldap_caches TSA_GUARDED_BY(mutex) ; - mutable AccessTokenCache access_token_cache TSA_GUARDED_BY(mutex) ; std::optional kerberos_params TSA_GUARDED_BY(mutex) ; std::unordered_map http_auth_servers TSA_GUARDED_BY(mutex) ; - std::unordered_map> jwt_validators TSA_GUARDED_BY(mutex) ; - std::unordered_map> token_processors TSA_GUARDED_BY(mutex) ; + mutable std::unordered_map> token_processors TSA_GUARDED_BY(mutex) ; + + struct TokenCacheEntry + { + std::chrono::system_clock::time_point expires_at; + String user_name; + std::set external_roles; + }; + + /// Home-made simple bi-mapping, needed to effectively clean up cache from old tokens. + using TokenToUsernameCache = std::unordered_map; // Access token -> cache entry + using UsernameToTokenCache = std::unordered_map; // User name -> access token + + mutable TokenToUsernameCache access_token_to_username_cache TSA_GUARDED_BY(mutex) ; + mutable UsernameToTokenCache username_to_access_token_cache TSA_GUARDED_BY(mutex) ; + + bool checkCredentialsAgainstProcessor(const ITokenProcessor & processor, + const TokenCredentials & credentials) const TSA_REQUIRES(mutex); void resetImpl() TSA_REQUIRES(mutex); }; diff --git a/src/Access/JWTValidator.cpp b/src/Access/JWTValidator.cpp deleted file mode 100644 index 3b9763448f05..000000000000 --- a/src/Access/JWTValidator.cpp +++ /dev/null @@ -1,367 +0,0 @@ -#include "JWTValidator.h" - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int AUTHENTICATION_FAILED; - extern const int INVALID_CONFIG_PARAMETER; -} - -namespace -{ - -bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path); -bool check_claims(const picojson::value::object & claims, const picojson::value::object & payload, const String & path) -{ - for (const auto & it : claims) - { - const auto & payload_it = payload.find(it.first); - if (payload_it == payload.end()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "Key '{}.{}' not found in JWT payload", path, it.first); - return false; - } - if (!check_claims(it.second, payload_it->second, path + "." + it.first)) - { - return false; - } - } - return true; -} - -bool check_claims(const picojson::value::array & claims, const picojson::value::array & payload, const String & path) -{ - if (claims.size() > payload.size()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload too small for claims key '{}'", path); - return false; - } - for (size_t claims_i = 0; claims_i < claims.size(); ++claims_i) - { - bool found = false; - const auto & claims_val = claims.at(claims_i); - for (const auto & payload_val : payload) - { - if (!check_claims(claims_val, payload_val, path + "[" + std::to_string(claims_i) + "]")) - continue; - found = true; - } - if (!found) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not contain an object matching claims key '{}[{}]'", path, claims_i); - return false; - } - } - return true; -} - -bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path) -{ - if (claims.is()) - { - if (!payload.is()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'array' in claims '{}'", path); - return false; - } - return check_claims(claims.get(), payload.get(), path); - } - if (claims.is()) - { - if (!payload.is()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'object' in claims '{}'", path); - return false; - } - return check_claims(claims.get(), payload.get(), path); - } - if (claims.is()) - { - if (!payload.is()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'bool' in claims '{}'", path); - return false; - } - if (claims.get() != payload.get()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); - return false; - } - return true; - } - if (claims.is()) - { - if (!payload.is()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'double' in claims '{}'", path); - return false; - } - if (claims.get() != payload.get()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); - return false; - } - return true; - } - if (claims.is()) - { - if (!payload.is()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'std::string' in claims '{}'", path); - return false; - } - if (claims.get() != payload.get()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); - return false; - } - return true; - } - #ifdef PICOJSON_USE_INT64 - if (claims.is()) - { - if (!payload.is()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'int64_t' in claims '{}'", path); - return false; - } - if (claims.get() != payload.get()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in claims '{}'. Expected '{}' but given '{}'", path, claims.get(), payload.get()); - return false; - } - return true; - } - #endif - LOG_ERROR(getLogger("JWTAuthentication"), "JWT claim '{}' does not match any known type", path); - return false; -} - -bool check_claims(const String & claims, const picojson::value::object & payload) -{ - if (claims.empty()) - return true; - picojson::value json; - auto errors = picojson::parse(json, claims); - if (!errors.empty()) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: {}", errors); - if (!json.is()) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: is not an object"); - return check_claims(json.get(), payload, ""); -} - -} - -bool IJWTValidator::validate(const String & claims, const String & token, String & username) -{ - try - { - auto decoded_jwt = jwt::decode(token); - - validateImpl(decoded_jwt); - - if (!check_claims(claims, decoded_jwt.get_payload_json())) - return false; - - username = decoded_jwt.get_subject(); - - return true; - } - catch (const std::exception & ex) - { - LOG_TRACE(getLogger("JWTAuthentication"), "{}: Failed to validate JWT: {}", name, ex.what()); - return false; - } -} - -void SimpleJWTValidatorParams::validate() const -{ - if (algo == "ps256" || - algo == "ps384" || - algo == "ps512" || - algo == "ed25519" || - algo == "ed448" || - algo == "rs256" || - algo == "rs384" || - algo == "rs512" || - algo == "es256" || - algo == "es256k" || - algo == "es384" || - algo == "es512" ) - { - if (public_key.empty()) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `public_key` parameter required for {}", algo); - } - else if (algo == "hs256" || - algo == "hs384" || - algo == "hs512" ) - { - if (static_key.empty()) - throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `static_key` parameter required for {}", algo); - } - else if (algo != "none") - throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); -} - - -SimpleJWTValidator::SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_) - : IJWTValidator(name_), verifier(jwt::verify()) -{ - auto algo = params_.algo; - - if (algo == "none") - verifier = verifier.allow_algorithm(jwt::algorithm::none()); - else if (algo == "ps256") - verifier = verifier.allow_algorithm(jwt::algorithm::ps256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "ps384") - verifier = verifier.allow_algorithm(jwt::algorithm::ps384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "ps512") - verifier = verifier.allow_algorithm(jwt::algorithm::ps512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "ed25519") - verifier = verifier.allow_algorithm(jwt::algorithm::ed25519(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "ed448") - verifier = verifier.allow_algorithm(jwt::algorithm::ed448(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "rs256") - verifier = verifier.allow_algorithm(jwt::algorithm::rs256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "rs384") - verifier = verifier.allow_algorithm(jwt::algorithm::rs384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "rs512") - verifier = verifier.allow_algorithm(jwt::algorithm::rs512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "es256") - verifier = verifier.allow_algorithm(jwt::algorithm::es256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "es256k") - verifier = verifier.allow_algorithm(jwt::algorithm::es256k(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "es384") - verifier = verifier.allow_algorithm(jwt::algorithm::es384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo == "es512") - verifier = verifier.allow_algorithm(jwt::algorithm::es512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); - else if (algo.starts_with("hs")) - { - auto key = params_.static_key; - if (params_.static_key_in_base64) - key = base64Decode(key); - if (algo == "hs256") - verifier = verifier.allow_algorithm(jwt::algorithm::hs256(key)); - else if (algo == "hs384") - verifier = verifier.allow_algorithm(jwt::algorithm::hs384(key)); - else if (algo == "hs512") - verifier = verifier.allow_algorithm(jwt::algorithm::hs512(key)); - else - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); - } - else - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); -} - -void SimpleJWTValidator::validateImpl(const jwt::decoded_jwt & token) const -{ - verifier.verify(token); -} - -void JWKSValidator::validateImpl(const jwt::decoded_jwt & token) const -{ - auto jwk = provider->getJWKS().get_jwk(token.get_key_id()); - auto subject = token.get_subject(); - auto algo = Poco::toLower(token.get_algorithm()); - auto verifier = jwt::verify(); - String public_key; - - try - { - auto issuer = token.get_issuer(); - auto x5c = jwk.get_x5c_key_value(); - - if (!x5c.empty() && !issuer.empty()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "{}: Verifying {} with 'x5c' key", name, subject); - public_key = jwt::helper::convert_base64_der_to_pem(x5c); - } - } - catch (const jwt::error::claim_not_present_exception &) - { - LOG_TRACE(getLogger("JWTAuthentication"), "{}: issuer or x5c was not specified, skip verification against them", name); - } - catch (const std::bad_cast &) - { - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: invalid claim value type found, claims must be strings"); - } - - if (public_key.empty()) - { - LOG_TRACE(getLogger("JWTAuthentication"), "{}: `issuer` or `x5c` not present, verifying {} with RSA components", name, subject); - const auto modulus = jwk.get_jwk_claim("n").as_string(); - const auto exponent = jwk.get_jwk_claim("e").as_string(); - public_key = jwt::helper::create_public_key_from_rsa_components(modulus, exponent); - } - - if (jwk.has_algorithm() && Poco::toLower(jwk.get_algorithm()) != algo) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT validation error: `alg` in JWK does not match the algorithm used in JWT"); - - if (algo == "rs256") - verifier = verifier.allow_algorithm(jwt::algorithm::rs256(public_key, "", "", "")); - else if (algo == "rs384") - verifier = verifier.allow_algorithm(jwt::algorithm::rs384(public_key, "", "", "")); - else if (algo == "rs512") - verifier = verifier.allow_algorithm(jwt::algorithm::rs512(public_key, "", "", "")); - else - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); - - verifier = verifier.leeway(60UL); - verifier.verify(token); -} - - -std::unique_ptr IJWTValidator::parseJWTValidator( - const Poco::Util::AbstractConfiguration & config, - const String & prefix, - const String & name) -{ - if (config.hasProperty(prefix + ".algo")) - { - SimpleJWTValidatorParams params = {}; - params.algo = Poco::toLower(config.getString(prefix + ".algo")); - params.static_key = config.getString(prefix + ".static_key", ""); - params.static_key_in_base64 = config.getBool(prefix + ".static_key_in_base64", false); - params.public_key = config.getString(prefix + ".public_key", ""); - params.private_key = config.getString(prefix + ".private_key", ""); - params.public_key_password = config.getString(prefix + ".public_key_password", ""); - params.private_key_password = config.getString(prefix + ".private_key_password", ""); - params.validate(); - return std::make_unique(name, params); - } - - std::shared_ptr provider; - if (config.hasProperty(prefix + ".uri")) - { - provider = std::make_shared(config.getString(prefix + ".uri"), config.getInt(prefix + ".refresh_ms", 300000)); - } - else if (config.hasProperty(prefix + ".static_jwks") || config.hasProperty(prefix + ".static_jwks_file")) - { - StaticJWKSParams params{ - config.getString(prefix + ".static_jwks", ""), - config.getString(prefix + ".static_jwks_file", "") - }; - provider = std::make_shared(params); - } - else - throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Either JWKS or JWKS URI must be specified in configuration"); - - return std::make_unique(name, provider); -} - -} diff --git a/src/Access/JWTValidator.h b/src/Access/JWTValidator.h deleted file mode 100644 index 2ab3f164d010..000000000000 --- a/src/Access/JWTValidator.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -#include -#include - -#include "Access/HTTPAuthClient.h" - -#include -#include -#include - -namespace DB -{ - -class IJWTValidator -{ -public: - explicit IJWTValidator(const String & name_) : name(name_) {} - virtual bool validate(const String & claims, const String & token, String & username); - virtual ~IJWTValidator() = default; - - static std::unique_ptr parseJWTValidator( - const Poco::Util::AbstractConfiguration & config, - const String & prefix, - const String & name); - -protected: - virtual void validateImpl(const jwt::decoded_jwt & token) const = 0; - const String name; -}; - -struct SimpleJWTValidatorParams -{ - String algo; - String static_key; - bool static_key_in_base64; - String public_key; - String private_key; - String public_key_password; - String private_key_password; - void validate() const; -}; - -class SimpleJWTValidator : public IJWTValidator -{ -public: - explicit SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_); -private: - void validateImpl(const jwt::decoded_jwt & token) const override; - jwt::verifier verifier; -}; - -class JWKSValidator : public IJWTValidator -{ -public: - explicit JWKSValidator(const String & name_, std::shared_ptr provider_) - : IJWTValidator(name_), provider(provider_) {} - - explicit JWKSValidator(const String & name_, const String & uri, const size_t refresh_ms_) - : JWKSValidator(name_, std::make_shared(uri, refresh_ms_)) {} -private: - void validateImpl(const jwt::decoded_jwt & token) const override; - - std::shared_ptr provider; -}; -} diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index 388e3f13c0b5..d1ec8be0c0a9 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -191,7 +191,7 @@ bool TokenAccessStorage::areTokenCredentialsValidNoLock(const User & user, const return false; if (const auto * token_credentials = dynamic_cast(&credentials)) - return external_authenticators.checkAccessTokenCredentials(*token_credentials); + return external_authenticators.checkTokenCredentials(*token_credentials); return false; } @@ -346,7 +346,7 @@ std::optional TokenAccessStorage::authenticateImpl( const auto & token_credentials = typeid_cast(credentials); - if (!external_authenticators.checkAccessTokenCredentialsByExactProcessor(token_credentials, provider_name)) + if (!external_authenticators.checkTokenCredentials(token_credentials, provider_name)) { // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. if (throw_if_user_not_exists) diff --git a/src/Access/TokenProcessors.h b/src/Access/TokenProcessors.h new file mode 100644 index 000000000000..712363d0f6e8 --- /dev/null +++ b/src/Access/TokenProcessors.h @@ -0,0 +1,166 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace DB +{ + +class ITokenProcessor +{ +public: + explicit ITokenProcessor(const String & processor_name_, + const UInt64 & token_cache_lifetime_, + const String & username_claim_ = "sub", + const String & groups_claim_ = "groups") + : processor_name(processor_name_), token_cache_lifetime(token_cache_lifetime_), username_claim(username_claim_), groups_claim(groups_claim_) {} + virtual ~ITokenProcessor() = default; + + virtual bool resolveAndValidate(const TokenCredentials & credentials) const = 0; + + virtual bool checkClaims(const TokenCredentials &, const String &) { return true; } + + UInt64 getTokenCacheLifetime() const { return token_cache_lifetime; } + String getProcessorName() const { return processor_name; } + + static std::unique_ptr parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & processor_name); + +protected: + const String processor_name; + const UInt64 token_cache_lifetime; + const String username_claim; + const String groups_claim; + + std::set parseGroupsFromJsonArray(picojson::array groups_array) const; +}; + +class StaticKeyJwtProcessor : public ITokenProcessor +{ +public: + explicit StaticKeyJwtProcessor(const String & processor_name_, + const UInt64 & token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & claims_, + const String & algo, + const String & static_key, + bool static_key_in_base64, + const String & public_key, + const String & private_key, + const String & public_key_password, + const String & private_key_password); + + bool resolveAndValidate(const TokenCredentials & credentials) const override; + bool checkClaims(const TokenCredentials & credentials, const String & claims_to_check) override; + +private: + const String claims; + jwt::verifier verifier = jwt::verify(); +}; + + +class JwksJwtProcessor : public ITokenProcessor +{ +public: + explicit JwksJwtProcessor(const String & processor_name_, + const UInt64 & token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & claims_, + const size_t verifier_leeway_, + std::shared_ptr provider_) + : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_), + claims(claims_), provider(provider_), verifier_leeway(verifier_leeway_) {} + + explicit JwksJwtProcessor(const String & processor_name_, + const UInt64 & token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & claims_, + const size_t verifier_leeway_, + const String & jwks_uri_, + const size_t jwks_cache_lifetime_) + : JwksJwtProcessor(processor_name_, + token_cache_lifetime_, + username_claim_, + groups_claim_, + claims_, + verifier_leeway_, + std::make_shared(jwks_uri_, jwks_cache_lifetime_)) {} + + bool resolveAndValidate(const TokenCredentials & credentials) const override; + bool checkClaims(const TokenCredentials & credentials, const String & claims_to_check) override; + +private: + const String claims; + mutable jwt::verifier verifier = jwt::verify(); + std::shared_ptr provider; + const size_t verifier_leeway; +}; + +/// Opaque tokens + +class GoogleTokenProcessor : public ITokenProcessor +{ +public: + GoogleTokenProcessor(const String & processor_name_, + const UInt64 token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_) + : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_) {} + + bool resolveAndValidate(const TokenCredentials & credentials) const override; +}; + +class AzureTokenProcessor : public ITokenProcessor +{ +public: + AzureTokenProcessor(const String & processor_name_, + const UInt64 token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_) + : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_) {} + + bool resolveAndValidate(const TokenCredentials & credentials) const override; +}; + +class OpenIdTokenProcessor : public ITokenProcessor +{ +public: + /// Specify endpoints manually + OpenIdTokenProcessor(const String & processor_name_, + const UInt64 token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & userinfo_endpoint_, + const String & token_introspection_endpoint_, + const UInt64 & verifier_leeway_, + const String & jwks_uri_, + const UInt64 & jwks_cache_lifetime_); + + /// Obtain endpoints from openid-configuration URL + OpenIdTokenProcessor(const String & processor_name_, + const UInt64 token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & openid_config_endpoint_, + const UInt64 & verifier_leeway_, + const UInt64 & jwks_cache_lifetime_); + + bool resolveAndValidate(const TokenCredentials & credentials) const override; +private: + Poco::URI userinfo_endpoint; + Poco::URI token_introspection_endpoint; + + /// Access token is often a valid JWT, so we can validate it locally to avoid unnecesary network requests. + std::optional jwt_validator = std::nullopt; +}; + +} diff --git a/src/Access/TokenProcessorsJWT.cpp b/src/Access/TokenProcessorsJWT.cpp new file mode 100644 index 000000000000..6c30dcd962bd --- /dev/null +++ b/src/Access/TokenProcessorsJWT.cpp @@ -0,0 +1,386 @@ +#include "TokenProcessors.h" + +#include +#include + +namespace DB { + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +namespace +{ + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path); +bool check_claims(const picojson::value::object & claims, const picojson::value::object & payload, const String & path) +{ + for (const auto & it : claims) + { + const auto & payload_it = payload.find(it.first); + if (payload_it == payload.end()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "Key '{}.{}' not found in JWT payload", path, it.first); + return false; + } + if (!check_claims(it.second, payload_it->second, path + "." + it.first)) + { + return false; + } + } + return true; +} + +bool check_claims(const picojson::value::array & claims, const picojson::value::array & payload, const String & path) +{ + if (claims.size() > payload.size()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload too small for claims key '{}'", path); + return false; + } + for (size_t claims_i = 0; claims_i < claims.size(); ++claims_i) + { + bool found = false; + const auto & claims_val = claims.at(claims_i); + for (const auto & payload_val : payload) + { + if (!check_claims(claims_val, payload_val, path + "[" + std::to_string(claims_i) + "]")) + continue; + found = true; + } + if (!found) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not contain an object matching claims key '{}[{}]'", path, claims_i); + return false; + } + } + return true; +} + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path) +{ + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match key type 'array' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match key type 'object' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match key type 'bool' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match key type 'double' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match key type 'std::string' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } +#ifdef PICOJSON_USE_INT64 + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match key type 'int64_t' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "JWT payload does not match the value in claims '{}'. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } +#endif + LOG_ERROR(getLogger("TokenAuthentication"), "JWT claim '{}' does not match any known type", path); + return false; +} + +bool check_claims(const String & claims, const picojson::value::object & payload) +{ + if (claims.empty()) + return true; + picojson::value json; + auto errors = picojson::parse(json, claims); + if (!errors.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: {}", errors); + if (!json.is()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: is not an object"); + return check_claims(json.get(), payload, ""); +} + +} + +std::set ITokenProcessor::parseGroupsFromJsonArray(picojson::array groups_array) const +{ + std::set external_groups_names; + + for (const auto & group: groups_array) + { + if (group.is()) + external_groups_names.insert(group.get()); + } + + return external_groups_names; +} + +StaticKeyJwtProcessor::StaticKeyJwtProcessor(const String & processor_name_, + const UInt64 & token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & claims_, + const String & algo, + const String & static_key, + bool static_key_in_base64, + const String & public_key, + const String & private_key, + const String & public_key_password, + const String & private_key_password) + : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_), + claims(claims_) +{ + if (algo == "ps256" || + algo == "ps384" || + algo == "ps512" || + algo == "ed25519" || + algo == "ed448" || + algo == "rs256" || + algo == "rs384" || + algo == "rs512" || + algo == "es256" || + algo == "es256k" || + algo == "es384" || + algo == "es512" ) + { + if (public_key.empty()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "{}: Invalid token processor definition, `public_key` parameter required for {}", processor_name, algo); + } + else if (algo == "hs256" || + algo == "hs384" || + algo == "hs512" ) + { + if (static_key.empty()) + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "{}: Invalid token processor definition, `static_key` parameter required for {}", processor_name, algo); + } + else if (algo != "none") + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "{}: Invalid token processor definition, unknown algorithm {}", processor_name, algo); + + if (algo == "none") + verifier = verifier.allow_algorithm(jwt::algorithm::none()); + else if (algo == "ps256") + verifier = verifier.allow_algorithm(jwt::algorithm::ps256(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "ps384") + verifier = verifier.allow_algorithm(jwt::algorithm::ps384(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "ps512") + verifier = verifier.allow_algorithm(jwt::algorithm::ps512(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "ed25519") + verifier = verifier.allow_algorithm(jwt::algorithm::ed25519(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "ed448") + verifier = verifier.allow_algorithm(jwt::algorithm::ed448(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "es256") + verifier = verifier.allow_algorithm(jwt::algorithm::es256(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "es256k") + verifier = verifier.allow_algorithm(jwt::algorithm::es256k(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "es384") + verifier = verifier.allow_algorithm(jwt::algorithm::es384(public_key, private_key, public_key_password, private_key_password)); + else if (algo == "es512") + verifier = verifier.allow_algorithm(jwt::algorithm::es512(public_key, private_key, public_key_password, private_key_password)); + else if (algo.starts_with("hs")) + { + auto key = static_key; + if (static_key_in_base64) + key = base64Decode(key); + if (algo == "hs256") + verifier = verifier.allow_algorithm(jwt::algorithm::hs256(key)); + else if (algo == "hs384") + verifier = verifier.allow_algorithm(jwt::algorithm::hs384(key)); + else if (algo == "hs512") + verifier = verifier.allow_algorithm(jwt::algorithm::hs512(key)); + else + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "{}: Invalid token processor definition, unknown algorithm {}", processor_name, algo); + } + else + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "{}: Invalid token processor definition, unknown algorithm {}", processor_name, algo); + +} + +namespace +{ +bool checkUserClaims(const TokenCredentials & credentials, const String & claims_to_check) +{ + try { + auto decoded_jwt = jwt::decode(credentials.getToken()); + return check_claims(claims_to_check, decoded_jwt.get_payload_json()); + } + catch (const std::exception &) + { + return false; + } +} +} + +bool StaticKeyJwtProcessor::checkClaims(const TokenCredentials & credentials, const String & claims_to_check) +{ + return checkUserClaims(credentials, claims_to_check); +} + +bool JwksJwtProcessor::checkClaims(const TokenCredentials & credentials, const String & claims_to_check) +{ + return checkUserClaims(credentials, claims_to_check); +} + +bool StaticKeyJwtProcessor::resolveAndValidate(const TokenCredentials & credentials) const +{ + try + { + auto decoded_jwt = jwt::decode(credentials.getToken()); + verifier.verify(decoded_jwt); + + if (!check_claims(claims, decoded_jwt.get_payload_json())) + return false; + + if (!decoded_jwt.has_payload_claim(username_claim)) + { + LOG_ERROR(getLogger("TokenAuthentication"), "{}: Specified username_claim {} not found in token", processor_name, username_claim); + return false; + } + + const_cast(credentials).setUserName(decoded_jwt.get_payload_claim(username_claim).as_string()); + + if (decoded_jwt.has_payload_claim(groups_claim)) + const_cast(credentials).setGroups(parseGroupsFromJsonArray(decoded_jwt.get_payload_claim(groups_claim).as_array())); + else + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Specified groups_claim {} not found in token, no external roles will be mapped", processor_name, groups_claim); + + return true; + } + catch (const std::exception & ex) + { + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Failed to validate JWT: {}", processor_name, ex.what()); + return false; + } +} + +bool JwksJwtProcessor::resolveAndValidate(const TokenCredentials & credentials) const +{ + auto decoded_jwt = jwt::decode(credentials.getToken()); + + if (!decoded_jwt.has_payload_claim(username_claim)) + { + LOG_ERROR(getLogger("TokenAuthentication"), "{}: Specified username_claim not found in token", processor_name); + return false; + } + + auto jwk = provider->getJWKS().get_jwk(decoded_jwt.get_key_id()); + auto username = decoded_jwt.get_payload_claim(username_claim).as_string(); + auto algo = Poco::toLower(decoded_jwt.get_algorithm()); + String public_key; + + try + { + auto issuer = decoded_jwt.get_issuer(); + auto x5c = jwk.get_x5c_key_value(); + + if (!x5c.empty() && !issuer.empty()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Verifying {} with 'x5c' key", processor_name, username); + public_key = jwt::helper::convert_base64_der_to_pem(x5c); + } + } + catch (const jwt::error::claim_not_present_exception &) + { + LOG_TRACE(getLogger("TokenAuthentication"), "{}: issuer or x5c was not specified, skip verification against them", processor_name); + } + catch (const std::bad_cast &) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: invalid claim value type found, claims must be strings"); + } + + if (public_key.empty()) + { + LOG_TRACE(getLogger("TokenAuthentication"), "{}: `issuer` or `x5c` not present, verifying {} with RSA components", processor_name, username); + const auto modulus = jwk.get_jwk_claim("n").as_string(); + const auto exponent = jwk.get_jwk_claim("e").as_string(); + public_key = jwt::helper::create_public_key_from_rsa_components(modulus, exponent); + } + + if (jwk.has_algorithm() && Poco::toLower(jwk.get_algorithm()) != algo) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT validation error: `alg` in JWK does not match the algorithm used in JWT"); + + if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(public_key, "", "", "")); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(public_key, "", "", "")); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(public_key, "", "", "")); + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); + + verifier = verifier.leeway(verifier_leeway); + verifier.verify(decoded_jwt); + + if (!check_claims(claims, decoded_jwt.get_payload_json())) + return false; + + const_cast(credentials).setUserName(decoded_jwt.get_payload_claim(username_claim).as_string()); + + if (decoded_jwt.has_payload_claim(groups_claim)) + const_cast(credentials).setGroups(parseGroupsFromJsonArray(decoded_jwt.get_payload_claim(groups_claim).as_array())); + else + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Specified groups_claim {{ not found in token, no external roles will be mapped", processor_name, groups_claim); + + return true; +} + +} diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/TokenProcessorsOpaque.cpp similarity index 52% rename from src/Access/AccessTokenProcessor.cpp rename to src/Access/TokenProcessorsOpaque.cpp index b6cab369f76d..529c228426b4 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/TokenProcessorsOpaque.cpp @@ -1,12 +1,15 @@ -#include +#include "TokenProcessors.h" + #include #include -#include -#include +namespace DB { -namespace DB +namespace ErrorCodes { + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} namespace { @@ -80,119 +83,36 @@ namespace } } - -[[maybe_unused]] const Poco::URI GoogleAccessTokenProcessor::token_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/tokeninfo"); -const Poco::URI GoogleAccessTokenProcessor::user_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/userinfo"); - -const Poco::URI AzureAccessTokenProcessor::user_info_uri = Poco::URI("https://graph.microsoft.com/oidc/userinfo"); - - -std::unique_ptr IAccessTokenProcessor::parseTokenProcessor( - const Poco::Util::AbstractConfiguration & config, - const String & prefix, - const String & name) +bool GoogleTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) const { - /// TODO: maybe bind external user to the processor it was created with? - if (config.hasProperty(prefix + ".provider")) - { - String provider = Poco::toLower(config.getString(prefix + ".provider")); - - String email_regex_str = config.getString(prefix + ".email_filter", ""); - UInt64 cache_lifetime = config.getUInt64(prefix + ".cache_lifetime", 3600); - String username_claim = config.getString(prefix + ".username_claim", "sub"); - - if (provider == "google") - { - return std::make_unique(name, cache_lifetime, email_regex_str); - } - else if (provider == "azure") - { - if (!config.hasProperty(prefix + ".client_id")) - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, - "Could not parse access token processor {}: client_id must be specified", name); - - if (!config.hasProperty(prefix + ".tenant_id")) - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, - "Could not parse access token processor {}: tenant_id must be specified", name); - - String tenant_id_str = config.getString(prefix + ".tenant_id"); - - return std::make_unique(name, cache_lifetime, email_regex_str, tenant_id_str); - } - else if (provider == "openid") - { - bool is_auto = config.hasProperty(prefix + ".configuration_endpoint"); - bool is_manual = config.hasProperty(prefix + ".userinfo_endpoint") && - config.hasProperty(prefix + ".token_introspection_endpoint") && - (config.hasProperty(prefix + ".userinfo_endpoint") == config.hasProperty(prefix + ".token_introspection_endpoint")); - - if (is_auto && !is_manual) - { - return std::make_unique(name, - cache_lifetime, - email_regex_str, - config.getString(prefix + ".configuration_endpoint"), - config.getString(prefix + ".groups_claim", "")); - } - else if (!is_auto && is_manual) - { - return std::make_unique(name, - cache_lifetime, - email_regex_str, - config.getString(prefix + ".userinfo_endpoint"), - config.getString(prefix + ".token_introspection_endpoint"), - config.getString(prefix + ".jwks_uri"), - config.getString(prefix + ".groups_claim", "")); - } + const String & token = credentials.getToken(); - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse access token processor {}: " - "Either configuration_endpoint or both userinfo_endpoint and token_introspection_endpoint shall be specified", name); - } - else - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, - "Could not parse access token processor {}: unknown provider type {}", name, provider); - } + std::unordered_map user_info; + picojson::object user_info_json = getObjectFromURI(Poco::URI("https://www.googleapis.com/oauth2/v3/userinfo"), token); - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, - "Could not parse access token processor {}: provider name must be specified", name); -} + if (!user_info_json.contains("email")) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, + "{}: Specified username_claim {} not found in token", processor_name, username_claim); + bool has_email = user_info_json.contains("email"); + if (has_email) + user_info["email"] = getValueByKey(user_info_json, "email"); -bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) -{ - const String & token = credentials.getToken(); + user_info[username_claim] = getValueByKey(user_info_json, username_claim); - auto user_info = getUserInfo(token); String user_name = user_info[username_claim]; - bool has_email = user_info.contains("email"); - - if (email_regex.ok()) - { - if (!has_email) - { - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to validate {} by e-mail", name, user_name); - return false; - } - /// Additionally validate user email to match regex from config. - if (!RE2::FullMatch(user_info["email"], email_regex)) - { - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to authenticate user {}: e-mail address is not permitted.", name, user_name); - return false; - } - - } /// Credentials are passed as const everywhere up the flow, so we have to comply, /// in this case const_cast looks acceptable. const_cast(credentials).setUserName(user_name); - auto token_info = getObjectFromURI(Poco::URI(token_info_uri), token); + auto token_info = getObjectFromURI(Poco::URI("https://www.googleapis.com/oauth2/v3/tokeninfo"), token); if (token_info.contains("exp")) const_cast(credentials).setExpiresAt(std::chrono::system_clock::from_time_t((getValueByKey(token_info, "exp")))); /// Groups info can only be retrieved if user email is known. - /// If no email found in user info, we skip this step and there are no external groups for the user. + /// If no email found in user info, we skip this step and there are no external roles for the user. if (has_email) { std::set external_groups_names; @@ -204,8 +124,8 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre if (!groups_response.contains("memberships") || !groups_response["memberships"].is()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: Failed to get Google groups: invalid content in response from server", name); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: Failed to get Google groups: invalid content in response from server", processor_name); return true; } @@ -213,16 +133,16 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre { if (!group.is()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: Failed to get Google groups: invalid content in response from server", name); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: Failed to get Google groups: invalid content in response from server", processor_name); continue; } auto group_data = group.get(); String group_name = getValueByKey(group_data["groupKey"].get(), "id"); external_groups_names.insert(group_name); - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: User {}: new external group {}", name, user_name, group_name); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: User {}: new external group {}", processor_name, user_name, group_name); } const_cast(credentials).setGroups(external_groups_names); @@ -230,8 +150,8 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre catch (const Exception & e) { /// Could not get groups info. Log it and skip it. - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: Failed to get Google groups, no external roles will be mapped. reason: {}", name, e.what()); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: Failed to get Google groups, no external roles will be mapped. reason: {}", processor_name, e.what()); return true; } } @@ -239,25 +159,7 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre return true; } -std::unordered_map GoogleAccessTokenProcessor::getUserInfo(const String & token) const -{ - std::unordered_map user_info_map; - picojson::object user_info_json = getObjectFromURI(user_info_uri, token); - - try - { - user_info_map["email"] = getValueByKey(user_info_json, "email"); - user_info_map[username_claim] = getValueByKey(user_info_json, username_claim); - return user_info_map; - } - catch (std::runtime_error & e) - { - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Failed to get user info with token: {}", name, e.what()); - } -} - - -bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +bool AzureTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) const { /// Token is a JWT in this case, but we cannot directly verify it against Azure AD JWKS. /// We will not trust user data in this token except for 'exp' value to determine caching duration. @@ -269,7 +171,8 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred try { - String username = validateTokenAndGetUsername(token); + picojson::object user_info_json = getObjectFromURI(Poco::URI("https://graph.microsoft.com/oidc/userinfo"), token); + String username = getValueByKey(user_info_json, username_claim); if (!username.empty()) { /// Credentials are passed as const everywhere up the flow, so we have to comply, @@ -277,7 +180,7 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred const_cast(credentials).setUserName(username); } else - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username with token", name); + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Failed to get username with token", processor_name); } catch (...) @@ -290,8 +193,8 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred const_cast(credentials).setExpiresAt(jwt::decode(token).get_expires_at()); } catch (...) { - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: No expiration data found in a valid token, will use default cache lifetime", name); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: No expiration data found in a valid token, will use default cache lifetime", processor_name); } std::set external_groups_names; @@ -303,8 +206,8 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred if (!groups_response.contains("value") || !groups_response["value"].is()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: Failed to get Azure groups: invalid content in response from server", name); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: Failed to get Azure groups: invalid content in response from server", processor_name); return true; } @@ -315,22 +218,22 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred /// Got some invalid response. Ignore this, log this. if (!group.is()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: Failed to get Azure groups: invalid content in response from server", name); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: Failed to get Azure groups: invalid content in response from server", processor_name); continue; } auto group_data = group.get(); String group_name = getValueByKey(group_data, "id"); external_groups_names.insert(group_name); - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: User {}: new external group {}", name, credentials.getUserName(), group_name); + LOG_TRACE(getLogger("TokenAuthentication"), "{}: User {}: new external group {}", processor_name, credentials.getUserName(), group_name); } } catch (const Exception & e) { /// Could not get groups info. Log it and skip it. - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: Failed to get Azure groups, no external roles will be mapped. reason: {}", name, e.what()); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: Failed to get Azure groups, no external roles will be mapped. reason: {}", processor_name, e.what()); return true; } @@ -339,58 +242,64 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred return true; } -String AzureAccessTokenProcessor::validateTokenAndGetUsername(const String & token) const -{ - picojson::object user_info_json = getObjectFromURI(user_info_uri, token); - return getValueByKey(user_info_json, username_claim); -} - - -OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, - const UInt64 cache_invalidation_interval_, - const String & email_regex_str, - const String & userinfo_endpoint_, - const String & token_introspection_endpoint_, - const String & jwks_uri_, - const String & groups_claim_) - : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), - userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_), groups_claim(groups_claim_) +OpenIdTokenProcessor::OpenIdTokenProcessor(const String & processor_name_, + const UInt64 token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & userinfo_endpoint_, + const String & token_introspection_endpoint_, + const UInt64 & verifier_leeway_, + const String & jwks_uri_, + const UInt64 & jwks_cache_lifetime_) + : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_), + userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_) { if (!jwks_uri_.empty()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: JWKS URI set, local JWT processing will be attempted", name); - jwt_validator.emplace(name_ + "jwks_val", jwks_uri_, cache_invalidation_interval_); + LOG_TRACE(getLogger("TokenAuthentication"), "{}: JWKS URI set, local JWT processing will be attempted", processor_name_); + jwt_validator.emplace(processor_name_ + "jwks_val", + token_cache_lifetime_, + username_claim_, + groups_claim_, + "", + verifier_leeway_, + jwks_uri_, + jwks_cache_lifetime_); } } -OpenIDAccessTokenProcessor::OpenIDAccessTokenProcessor(const String & name_, - const UInt64 cache_invalidation_interval_, - const String & email_regex_str, - const String & openid_config_endpoint_, - const String & groups_claim_) - : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) +OpenIdTokenProcessor::OpenIdTokenProcessor(const String & processor_name_, + const UInt64 token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_, + const String & openid_config_endpoint_, + const UInt64 & verifier_leeway_, + const UInt64 & jwks_cache_lifetime_) + : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_) { const picojson::object openid_config = getObjectFromURI(Poco::URI(openid_config_endpoint_)); if (!openid_config.contains("userinfo_endpoint") || !openid_config.contains("introspection_endpoint")) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Cannot extract userinfo_endpoint or introspection_endpoint from OIDC configuration, consider manual configuration.", name); + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Cannot extract userinfo_endpoint or introspection_endpoint from OIDC configuration, consider manual configuration.", processor_name); - OpenIDAccessTokenProcessor(name_, - cache_invalidation_interval_, - email_regex_str, + OpenIdTokenProcessor(processor_name_, + token_cache_lifetime_, + username_claim_, + groups_claim_, getValueByKey(openid_config, "userinfo_endpoint"), getValueByKey(openid_config, "introspection_endpoint"), + verifier_leeway_, openid_config.contains("jwks_uri") ? getValueByKey(openid_config, "jwks_uri") : "", - groups_claim_); + jwks_cache_lifetime_); } -bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +bool OpenIdTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) const { const String & token = credentials.getToken(); String username; picojson::object user_info_json; - if (jwt_validator.has_value() && jwt_validator.value().validate("", token, username)) + if (jwt_validator.has_value() && jwt_validator.value().resolveAndValidate(credentials)) { try { @@ -403,7 +312,7 @@ bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre } catch (const std::exception & ex) { - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to process token as JWT: {}", name, ex.what()); + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Failed to process token as JWT: {}", processor_name, ex.what()); } } @@ -423,12 +332,12 @@ bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre if (user_info_json.empty()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to obtain user info", name); + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Failed to obtain user info", processor_name); return false; } else if (username.empty()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username", name); + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Failed to get username", processor_name); return false; } @@ -442,8 +351,8 @@ bool OpenIDAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre { if (!user_info_json[groups_claim].is()) { - LOG_TRACE(getLogger("AccessTokenProcessor"), - "{}: Failed to extract groups: invalid content in user data", name); + LOG_TRACE(getLogger("TokenAuthentication"), + "{}: Failed to extract groups: invalid content in user data", processor_name); return true; } diff --git a/src/Access/TokenProcessorsParse.cpp b/src/Access/TokenProcessorsParse.cpp new file mode 100644 index 000000000000..5611e5c59455 --- /dev/null +++ b/src/Access/TokenProcessorsParse.cpp @@ -0,0 +1,100 @@ +#include "TokenProcessors.h" + +#include +#include + +namespace DB { + +namespace ErrorCodes +{ + extern const int INVALID_CONFIG_PARAMETER; +} + +std::unique_ptr ITokenProcessor::parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & processor_name) +{ + auto token_cache_lifetime = config.getUInt64(prefix + ".token_cache_lifetime", 3600); + auto username_claim = config.getString(prefix + ".username_claim", "sub"); + auto groups_claim = config.getString(prefix + ".groups_claim", "groups"); + + if (config.hasProperty(prefix + ".provider")) // One of providers for opaque tokens processing + { + auto provider_type = Poco::toLower(config.getString(prefix + ".provider")); + + if (provider_type == "google") + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim); + } + else if (provider_type == "azure") + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim); + } + else if (provider_type == "openid") + { + auto verifier_leeway = config.getUInt64(prefix + ".verifier_leeway", 60); + auto jwks_cache_lifetime = config.getUInt64(prefix + ".jwks_cache_lifetime", 3600); + + if (config.hasProperty(prefix + ".configuration_endpoint")) + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, + groups_claim, + config.getString(prefix + ".openid_config_endpoint"), + verifier_leeway, + jwks_cache_lifetime); + } + else + { + auto userinfo_endpoint = config.getString(prefix + ".userinfo_endpoint"); + auto token_introspection_endpoint = config.getString(prefix + ".token_introspection_endpoint"); + auto jwks_uri = config.getString(prefix + ".jwks_uri", ""); + return std::make_unique(processor_name, token_cache_lifetime, username_claim, + groups_claim, + config.getString(prefix + ".userinfo_endpoint"), + config.getString(prefix + ".token_introspection_endpoint"), + verifier_leeway, + config.getString(prefix + ".jwks_uri", ""), + jwks_cache_lifetime); + } + } + } + else if (config.hasProperty(prefix + ".algo")) /// StaticKeyJwtProcessor + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".claims", ""), + Poco::toLower(config.getString(prefix + ".algo")), + config.getString(prefix + ".static_key", ""), + config.getBool(prefix + ".static_key_in_base64", false), + config.getString(prefix + ".public_key", ""), + config.getString(prefix + ".private_key", ""), + config.getString(prefix + ".public_key_password", ""), + config.getString(prefix + ".private_key_password", "")); + } + else if (config.hasProperty(prefix + ".static_jwks") || config.hasProperty(prefix + ".static_jwks_file")) + { + StaticJWKSParams params + { + config.getString(prefix + ".static_jwks", ""), + config.getString(prefix + ".static_jwks_file", "") + }; + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".claims", ""), + config.getUInt64(prefix + ".verifier_leeway", 0), + std::make_shared(params)); + } + else if (config.hasProperty(prefix + ".jwks_uri")) + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".claims", ""), + config.getUInt64(prefix + ".verifier_leeway", 0), + config.getString(prefix + ".jwks_uri"), + config.getUInt(prefix + ".jwks_cache_lifetime", 300000)); + } + + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse token processor"); +} + + + +} diff --git a/src/Server/HTTP/authenticateUserByHTTP.cpp b/src/Server/HTTP/authenticateUserByHTTP.cpp index 07942ef43ab6..28b2139c522f 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.cpp +++ b/src/Server/HTTP/authenticateUserByHTTP.cpp @@ -220,8 +220,7 @@ bool authenticateUserByHTTP( const auto token_credentials = TokenCredentials(jwt_token.substr(BEARER_PREFIX.length())); const auto & external_authenticators = global_context->getAccessControl().getExternalAuthenticators(); - if (!external_authenticators.resolveJWTCredentials(token_credentials, false) - && !external_authenticators.checkAccessTokenCredentials(token_credentials)) + if (!external_authenticators.checkTokenCredentials(token_credentials)) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: Token could not be verified."); current_credentials = std::make_unique(token_credentials); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 532a1e4eace4..a36ffb2d6878 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1865,8 +1865,7 @@ void TCPHandler::receiveHello() const auto & external_authenticators = server.context()->getAccessControl().getExternalAuthenticators(); - if (!external_authenticators.resolveJWTCredentials(credentials, false)) - external_authenticators.checkAccessTokenCredentials(credentials); + external_authenticators.checkTokenCredentials(credentials); session->authenticate(credentials, getClientAddress(client_info)); return; From 8d9928ffe067fb8c988592e611fa56692714df63 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Sun, 3 Aug 2025 20:40:08 +0000 Subject: [PATCH 09/24] tidy fix, cleanup --- src/Access/TokenProcessors.h | 34 ++++++++++++++-------------- src/Access/TokenProcessorsJWT.cpp | 2 +- src/Access/TokenProcessorsOpaque.cpp | 12 +++++----- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Access/TokenProcessors.h b/src/Access/TokenProcessors.h index 712363d0f6e8..b669e171493b 100644 --- a/src/Access/TokenProcessors.h +++ b/src/Access/TokenProcessors.h @@ -14,7 +14,7 @@ class ITokenProcessor { public: explicit ITokenProcessor(const String & processor_name_, - const UInt64 & token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_ = "sub", const String & groups_claim_ = "groups") : processor_name(processor_name_), token_cache_lifetime(token_cache_lifetime_), username_claim(username_claim_), groups_claim(groups_claim_) {} @@ -45,7 +45,7 @@ class StaticKeyJwtProcessor : public ITokenProcessor { public: explicit StaticKeyJwtProcessor(const String & processor_name_, - const UInt64 & token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & claims_, @@ -70,23 +70,23 @@ class JwksJwtProcessor : public ITokenProcessor { public: explicit JwksJwtProcessor(const String & processor_name_, - const UInt64 & token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & claims_, - const size_t verifier_leeway_, + size_t verifier_leeway_, std::shared_ptr provider_) : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_), claims(claims_), provider(provider_), verifier_leeway(verifier_leeway_) {} explicit JwksJwtProcessor(const String & processor_name_, - const UInt64 & token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & claims_, - const size_t verifier_leeway_, + size_t verifier_leeway_, const String & jwks_uri_, - const size_t jwks_cache_lifetime_) + size_t jwks_cache_lifetime_) : JwksJwtProcessor(processor_name_, token_cache_lifetime_, username_claim_, @@ -111,9 +111,9 @@ class GoogleTokenProcessor : public ITokenProcessor { public: GoogleTokenProcessor(const String & processor_name_, - const UInt64 token_cache_lifetime_, - const String & username_claim_, - const String & groups_claim_) + UInt64 token_cache_lifetime_, + const String & username_claim_, + const String & groups_claim_) : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_) {} bool resolveAndValidate(const TokenCredentials & credentials) const override; @@ -123,7 +123,7 @@ class AzureTokenProcessor : public ITokenProcessor { public: AzureTokenProcessor(const String & processor_name_, - const UInt64 token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_) : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_) {} @@ -136,23 +136,23 @@ class OpenIdTokenProcessor : public ITokenProcessor public: /// Specify endpoints manually OpenIdTokenProcessor(const String & processor_name_, - const UInt64 token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & userinfo_endpoint_, const String & token_introspection_endpoint_, - const UInt64 & verifier_leeway_, + UInt64 verifier_leeway_, const String & jwks_uri_, - const UInt64 & jwks_cache_lifetime_); + UInt64 jwks_cache_lifetime_); /// Obtain endpoints from openid-configuration URL OpenIdTokenProcessor(const String & processor_name_, - const UInt64 token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & openid_config_endpoint_, - const UInt64 & verifier_leeway_, - const UInt64 & jwks_cache_lifetime_); + UInt64 verifier_leeway_, + UInt64 jwks_cache_lifetime_); bool resolveAndValidate(const TokenCredentials & credentials) const override; private: diff --git a/src/Access/TokenProcessorsJWT.cpp b/src/Access/TokenProcessorsJWT.cpp index 6c30dcd962bd..170bb942f4fa 100644 --- a/src/Access/TokenProcessorsJWT.cpp +++ b/src/Access/TokenProcessorsJWT.cpp @@ -170,7 +170,7 @@ std::set ITokenProcessor::parseGroupsFromJsonArray(picojson::array group } StaticKeyJwtProcessor::StaticKeyJwtProcessor(const String & processor_name_, - const UInt64 & token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & claims_, diff --git a/src/Access/TokenProcessorsOpaque.cpp b/src/Access/TokenProcessorsOpaque.cpp index 529c228426b4..0f71c47ccc59 100644 --- a/src/Access/TokenProcessorsOpaque.cpp +++ b/src/Access/TokenProcessorsOpaque.cpp @@ -243,14 +243,14 @@ bool AzureTokenProcessor::resolveAndValidate(const TokenCredentials & credential } OpenIdTokenProcessor::OpenIdTokenProcessor(const String & processor_name_, - const UInt64 token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & userinfo_endpoint_, const String & token_introspection_endpoint_, - const UInt64 & verifier_leeway_, + UInt64 verifier_leeway_, const String & jwks_uri_, - const UInt64 & jwks_cache_lifetime_) + UInt64 jwks_cache_lifetime_) : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_), userinfo_endpoint(userinfo_endpoint_), token_introspection_endpoint(token_introspection_endpoint_) { @@ -269,12 +269,12 @@ OpenIdTokenProcessor::OpenIdTokenProcessor(const String & processor_name_, } OpenIdTokenProcessor::OpenIdTokenProcessor(const String & processor_name_, - const UInt64 token_cache_lifetime_, + UInt64 token_cache_lifetime_, const String & username_claim_, const String & groups_claim_, const String & openid_config_endpoint_, - const UInt64 & verifier_leeway_, - const UInt64 & jwks_cache_lifetime_) + UInt64 verifier_leeway_, + UInt64 jwks_cache_lifetime_) : ITokenProcessor(processor_name_, token_cache_lifetime_, username_claim_, groups_claim_) { const picojson::object openid_config = getObjectFromURI(Poco::URI(openid_config_endpoint_)); From 59ccf744eb61b12373ba3ff4a18fcb5a9f68cf7a Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Sun, 3 Aug 2025 23:08:41 +0000 Subject: [PATCH 10/24] fix build --- src/Access/TokenProcessorsOpaque.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Access/TokenProcessorsOpaque.cpp b/src/Access/TokenProcessorsOpaque.cpp index 0f71c47ccc59..9a458f0d8694 100644 --- a/src/Access/TokenProcessorsOpaque.cpp +++ b/src/Access/TokenProcessorsOpaque.cpp @@ -282,15 +282,18 @@ OpenIdTokenProcessor::OpenIdTokenProcessor(const String & processor_name_, if (!openid_config.contains("userinfo_endpoint") || !openid_config.contains("introspection_endpoint")) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Cannot extract userinfo_endpoint or introspection_endpoint from OIDC configuration, consider manual configuration.", processor_name); - OpenIdTokenProcessor(processor_name_, - token_cache_lifetime_, - username_claim_, - groups_claim_, - getValueByKey(openid_config, "userinfo_endpoint"), - getValueByKey(openid_config, "introspection_endpoint"), - verifier_leeway_, - openid_config.contains("jwks_uri") ? getValueByKey(openid_config, "jwks_uri") : "", - jwks_cache_lifetime_); + if (!openid_config.contains("jwks_uri")) + { + LOG_TRACE(getLogger("TokenAuthentication"), "{}: JWKS URI set, local JWT processing will be attempted", processor_name_); + jwt_validator.emplace(processor_name_ + "jwks_val", + token_cache_lifetime_, + username_claim_, + groups_claim_, + "", + verifier_leeway_, + getValueByKey(openid_config, "jwks_uri"), + jwks_cache_lifetime_); + } } bool OpenIdTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) const From d5f24f9c7f913228aa0dc1d62329ee4d6119db2b Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Sun, 3 Aug 2025 23:41:07 +0000 Subject: [PATCH 11/24] upload new doc version --- .../operations/external-authenticators/jwt.md | 182 ------------------ .../external-authenticators/tokens.md | 176 +++++++++++++---- 2 files changed, 143 insertions(+), 215 deletions(-) delete mode 100644 docs/en/operations/external-authenticators/jwt.md diff --git a/docs/en/operations/external-authenticators/jwt.md b/docs/en/operations/external-authenticators/jwt.md deleted file mode 100644 index 4b44239f8f37..000000000000 --- a/docs/en/operations/external-authenticators/jwt.md +++ /dev/null @@ -1,182 +0,0 @@ ---- -slug: /en/operations/external-authenticators/jwt ---- -# JWT -import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; - - - -Existing and properly configured ClickHouse users can be authenticated via JWT. - -Currently, JWT can only be used as an external authenticator for existing users, which are defined in `users.xml` or in local access control paths. -The username will be extracted from the JWT after validating the token expiration and against the signature. Signature can be validated by: -- static public key -- static JWKS -- received from the JWKS servers - -It is mandatory for a JWT tot indicate the name of the ClickHouse user under `"sub"` claim, otherwise it will not be accepted. - -A JWT may additionally be verified by checking the JWT payload. -In this case, the occurrence of specified claims from the user settings in the JWT payload is checked. -See [Enabling JWT authentication in `users.xml`](#enabling-jwt-auth-in-users-xml) - -To use JWT authentication, JWT validators must be configured in ClickHouse config. - - -## Enabling JWT validators in ClickHouse {#enabling-jwt-validators-in-clickhouse} - -To enable JWT validators, add `token_validators` section in `config.xml`. This section may contain several JWT verifiers, minimum is 1. - -### Verifying JWT signature using static key {$verifying-jwt-signature-using-static-key} - -**Example** -```xml - - - - - HS256 - my_static_secret - - - -``` - -#### Parameters: - -- `algo` - Algorithm for validate signature. Supported: - - | HMAC | RSA | ECDSA | PSS | EdDSA | - |-------| ----- | ------ | ----- | ------- | - | HS256 | RS256 | ES256 | PS256 | Ed25519 | - | HS384 | RS384 | ES384 | PS384 | Ed448 | - | HS512 | RS512 | ES512 | PS512 | | - | | | ES256K | | | - Also support None. -- `static_key` - key for symmetric algorithms. Mandatory for `HS*` family algorithms. -- `static_key_in_base64` - indicates if the `static_key` key is base64-encoded. Optional, default: `False`. -- `public_key` - public key for asymmetric algorithms. Mandatory except for `HS*` family algorithms and `None`. -- `private_key` - private key for asymmetric algorithms. Optional. -- `public_key_password` - public key password. Optional. -- `private_key_password` - private key password. Optional. - -### Verifying JWT signature using static JWKS {$verifying-jwt-signature-using-static-jwks} - -:::note -Only RS* family algorithms are supported! -::: - -**Example** -```xml - - - - - {"keys": [{"kty": "RSA", "alg": "RS256", "kid": "mykid", "n": "_public_key_mod_", "e": "AQAB"}]} - - - -``` - -#### Parameters: -- `static_jwks` - content of JWKS in json -- `static_jwks_file` - path to file with JWKS - -:::note -Only one of `static_jwks` or `static_jwks_file` keys must be present in one verifier -::: - -### Verifying JWT signature using JWKS servers {$verifying-jwt-signature-using-static-jwks} - -**Example** -```xml - - - - - http://localhost:8000/.well-known/jwks.json - 300000 - - - -``` - -#### Parameters: - -- `uri` - JWKS endpoint. Mandatory. -- `jwks_refresh_timeout` - Period for resend request for refreshing JWKS. Optional, default: 300000. - -### Enabling JWT authentication in `users.xml` {#enabling-jwt-auth-in-users-xml} - -In order to enable JWT authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. - -Parameters: -- `claims` - An optional string containing a json object that should be contained in the token payload. - -Example (goes into `users.xml`): -```xml - - - - - - {"resource_access":{"account": {"roles": ["view-profile"]}}} - - - -``` - -Here, the JWT payload must contain `["view-profile"]` on path `resource_access.account.roles`, otherwise authentication will not succeed even with a valid JWT. - -``` -{ -... - "resource_access": { - "account": { - "roles": ["view-profile"] - } - }, -... -} -``` - -:::note -JWT authentication cannot be used together with any other authentication method. The presence of any other sections like `password` alongside `jwt` will force ClickHouse to shut down. -::: - -### Enabling JWT authentication using SQL {#enabling-jwt-auth-using-sql} - -When [SQL-driven Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled in ClickHouse, users identified by JWT authentication can also be created using SQL statements. - -```sql -CREATE USER my_user IDENTIFIED WITH jwt CLAIMS '{"resource_access":{"account": {"roles": ["view-profile"]}}}' -``` - -Or without additional JWT payload checks: - -```sql -CREATE USER my_user IDENTIFIED WITH jwt -``` - -## JWT authentication examples {#jwt-authentication-examples} - -#### Console client - -``` -clickhouse-client -jwt -``` - -#### HTTP requests - -``` -curl 'http://localhost:8080/?' \ - -H 'Authorization: Bearer ' \ - -H 'Content type: text/plain;charset=UTF-8' \ - --data-raw 'SELECT current_user()' -``` -:::note -ClickHouse will look for a JWT token in (by priority): -1. `X-ClickHouse-JWT-Token` header. -2. `Authorization` header. -3. `token` request parameter. In this case, the "Bearer" prefix should not exist. -::: diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 7a2a2683bcd2..59da8b615bee 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -1,29 +1,117 @@ --- slug: /en/operations/external-authenticators/oauth -title: "OAuth 2.0" +title: "Token-based authentication" --- import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; -OAuth 2.0 access tokens can be used to authenticate ClickHouse users. This works in two ways: +ClickHouse users can be authenticated using tokens. This works in two ways: -- Existing users (defined in `users.xml` or in local access control paths) can be authenticated with access token if this user can be `IDENTIFIED WITH jwt`. -- Use Identity Provider (IdP) as an external user directory and allow locally undefined users to be authenticated with a token if it is valid and recognized by the provider. +- Existing users (defined in `users.xml` or in local access control paths) can be authenticated with a token if this user can be `IDENTIFIED WITH jwt`. +- Use the information from the token or from an external Identity Provider (IdP) as a source of user definitions and allow locally undefined users to be authenticated with a valid token. -Though this authentication method is different from JWT authentication, it works under the same authentication method to maintain better compatibility. +Although not all tokens are JWTs, under the hood both ways are treated as the same authentication method to maintain better compatibility. -For both of these approaches a definition of `token_processors` is mandatory. +## Token Processors -## Access Token Processors +To use token-based authentication, add `token_processors` section to `config.xml` and define at least one token processor in it. Its contents are different for different token processing workflows (JWT and opaque (generic) tokens). -To define an access token processor, add `token_processors` section to `config.xml`. Example: +### Common configuration parameters +In addition to specific parameters (see below), there are also parameters that can be applied to a token processor of any type: +- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. +- `username_claim` -- name of claim (field) that will be treated as ClickHouse username. Optional, default: "sub". +- `groups_claim` -- Name of claim (field) that contains list of groups user belongs to. This claim will be looked up in the token itself (in case token is a valid JWT, e.g. in Keycloak) or in response from `/userinfo`. Optional, default: "groups". + +### JWT (JSON Web Token) + +JWT itself is a source of information about user. It can be decoded locally, and its integrity can be verified using token issuer's (public) key. + +This key can exist in three ways: +* ##### Static key: +```xml + + + + HS256 + my_static_secret + + + +``` +**Parameters:** +- `algo` - Algorithm for validate signature. Mandatory. Supported values: + + | HMAC | RSA | ECDSA | PSS | EdDSA | + |-------| ----- | ------ | ----- | ------- | + | HS256 | RS256 | ES256 | PS256 | Ed25519 | + | HS384 | RS384 | ES384 | PS384 | Ed448 | + | HS512 | RS512 | ES512 | PS512 | | + | | | ES256K | | | + Also supports None (though not recommended). +- `static_key` - key for symmetric algorithms. Mandatory for `HS*` family algorithms. +- `static_key_in_base64` - indicates if the `static_key` key is base64-encoded. Optional, default: `False`. +- `public_key` - public key for asymmetric algorithms. Mandatory except for `HS*` family algorithms and `None`. +- `private_key` - private key for asymmetric algorithms. Optional. +- `public_key_password` - public key password. Optional. +- `private_key_password` - private key password. Optional. + +* ##### Static JWKS +```xml + + + + {"keys": [{"kty": "RSA", "alg": "RS256", "kid": "mykid", "n": "_public_key_mod_", "e": "AQAB"}]} + + + +``` + +#### Parameters: +- `static_jwks` - content of JWKS in JSON +- `static_jwks_file` - path to a file with JWKS +- `claims` - A string containing a JSON object that should be contained in the token payload. If this parameter is defined, token without corresponding payload will be considered invalid. Optional. +- `verifier_leeway` - Clock skew tolerance (seconds). Useful for handling small differences in system clocks between ClickHouse and the token issuer. Optional. + +:::note +Only one of `static_jwks` or `static_jwks_file` keys must be present in one verifier +::: + +:::note +Only RS* family algorithms are supported! +::: + +* ##### Remote JWKS +```xml + + + + http://localhost:8000/.well-known/jwks.json + 300000 + + + +``` + +#### Parameters: + +- `uri` - JWKS endpoint. Mandatory. +- `jwks_refresh_timeout` - Period for resend request for refreshing JWKS. Optional, default: 300000. +- `claims` - A string containing a JSON object that should be contained in the token payload. If this parameter is defined, token without corresponding payload will be considered invalid. Optional. +- `verifier_leeway` - Clock skew tolerance (seconds). Useful for handling small differences in system clocks between ClickHouse and the token issuer. Optional. + + +### Opaque tokens + +To define a token processor, add `token_processors` section to `config.xml`. Example: ```xml - azure - claim_name + openid + 600 + sub + groups @@ -36,51 +124,73 @@ Different providers have different sets of parameters. **Parameters** - `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure", "OpenID". -- `username_claim` -- name of claim (field) that will be treated as ClickHouse user name. Optional, default: "sub". -- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. -- `groups_claim` -- Name of claim (field) that contains list of groups user belongs to. This claim will be looked up in the token itself (in case token is a valid JWT, e.g. in Keycloak) or in response from `/userinfo`. Optional parameter. -- `configuration_endpoint` -- URI of `.well-known/openid-configuration`. Optional parameter, useful only for OIDC-compliant providers (e.g. Keycloak). -- `userinfo_endpoint` -- URI of userinfo endpoint. Optional parameter. -- `token_introspection_endpoint` -- URI of token introspection endpoint. Optional parameter. +- `configuration_endpoint` -- URI of `.well-known/openid-configuration`. Optional parameter, used only for OpenID providers. +- `userinfo_endpoint` -- URI of userinfo endpoint. Optional parameter. Optional parameter, used only for OpenID providers. +- `token_introspection_endpoint` -- URI of token introspection endpoint. Optional parameter. Optional parameter, used only for OpenID providers. :::note -Either `configuration_endpoint` or both `userinfo_endpoint` and `token_introspection_endpoint` shall be set. If none of them are set or all three are set, this is invalid configuration, it will not be parsed. +Either `configuration_endpoint` or both `userinfo_endpoint` and `token_introspection_endpoint` shall be set. If none of them are set or all three are set, this is an invalid configuration that will not be parsed. ::: + ### Tokens cache To reduce number of requests to IdP, tokens are cached internally for no longer then `cache_lifetime` seconds. If token expires sooner than `cache_lifetime`, then cache entry for this token will only be valid while token is valid. If token lifetime is longer than `cache_lifetime`, cache entry for this token will be valid for `cache_lifetime`. -## IdP as External Authenticator {#idp-external-authenticator} +## Enabling token authentication for a user in `users.xml` {#enabling-jwt-auth-in-users-xml} + +In order to enable token-based authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. -Locally defined users can be authenticated with an access token. To allow this, `jwt` must be specified as user's authentication method. Example: +Parameters: +- `claims` - An optional string containing a json object that should be contained in the token payload. +Example (goes into `users.xml`): ```xml - - - - - - - - - + + + {"resource_access":{"account": {"roles": ["view-profile"]}}} + + ``` -Inside `jwt` one or more specific access token processors names can be specified -- only those processors will be tried when authenticating. If no processors are specified, _all_ processors will be tried. +Here, the JWT payload must contain `["view-profile"]` on path `resource_access.account.roles`, otherwise authentication will not succeed even with a valid JWT. -At each login attempt, ClickHouse will attempt to validate token and get user info against every defined access token provider. +:::note +If `claims` is defined, this user will not be able to authenticate using opaque tokens, so, only JWT-based authentication will be available. +::: + +``` +{ +... + "resource_access": { + "account": { + "roles": ["view-profile"] + } + }, +... +} +``` + +:::note +JWT authentication cannot be used together with any other authentication method. The presence of any other sections like `password` alongside `jwt` will force ClickHouse to shut down. +::: + +## Enabling token authentication using SQL {#enabling-jwt-auth-using-sql} -When SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled, users that are authenticated with tokens can also be created using the [CREATE USER](/docs/en/sql-reference/statements/create/user.md#create-user-statement) statement. +When [SQL-driven Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled in ClickHouse, users identified by JWT authentication can also be created using SQL statements. + +```sql +CREATE USER my_user IDENTIFIED WITH jwt CLAIMS '{"resource_access":{"account": {"roles": ["view-profile"]}}}' +``` -Query: +Or without additional JWT payload checks: ```sql -CREATE USER my_user IDENTIFIED WITH jwt; +CREATE USER my_user IDENTIFIED WITH jwt ``` ## Identity Provider as an External User Directory {#idp-external-user-directory} From fa17b22c66923e62591fb978f703b0212c96bb5e Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Sun, 3 Aug 2025 23:47:05 +0000 Subject: [PATCH 12/24] one more fix --- src/Access/TokenAccessStorage.cpp | 2 +- src/Access/TokenProcessorsParse.cpp | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index d1ec8be0c0a9..a68d89b3c3a4 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -301,7 +301,7 @@ void TokenAccessStorage::assignRolesNoLock(User & user, const std::set & if (external_roles.empty()) roles_per_users.erase(user_name); else - roles_per_users[user_name] = std::move(external_roles); + roles_per_users[user_name] = external_roles; external_role_hashes[user_name] = external_roles_hash; } diff --git a/src/Access/TokenProcessorsParse.cpp b/src/Access/TokenProcessorsParse.cpp index 5611e5c59455..dc0f8a16b095 100644 --- a/src/Access/TokenProcessorsParse.cpp +++ b/src/Access/TokenProcessorsParse.cpp @@ -46,9 +46,6 @@ std::unique_ptr ITokenProcessor::parseTokenProcessor( } else { - auto userinfo_endpoint = config.getString(prefix + ".userinfo_endpoint"); - auto token_introspection_endpoint = config.getString(prefix + ".token_introspection_endpoint"); - auto jwks_uri = config.getString(prefix + ".jwks_uri", ""); return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, config.getString(prefix + ".userinfo_endpoint"), From 47894168fd3e42be3964708b7613b353b83e61f1 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Mon, 4 Aug 2025 12:36:12 +0000 Subject: [PATCH 13/24] minor docs update + exception fix --- docs/en/operations/external-authenticators/index.md | 3 +-- docs/en/operations/external-authenticators/tokens.md | 4 +++- src/Access/ExternalAuthenticators.cpp | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index 25d08d03fb7c..1391b7d14ce9 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -19,5 +19,4 @@ The following external authenticators and directories are supported: - Kerberos [Authenticator](/operations/external-authenticators/kerberos#kerberos-as-an-external-authenticator-for-existing-users) - [SSL X.509 authentication](/operations/external-authenticators/ssl-x509) - HTTP [Authenticator](./http.md) -- JWT [Authenticator](./jwt.md) -- Access Token [Authenticator](./tokens.md) +- Token-based [Authenticator](./tokens.md) diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 59da8b615bee..5e7a20749fa1 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -213,7 +213,9 @@ All this implies that the SQL-driven [Access Control and Account Management](/do - + + \bclickhouse-[a-zA-Z0-9]+\b + diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index a05ce5874340..9b166c3a4d89 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -649,7 +649,7 @@ bool ExternalAuthenticators::checkTokenCredentials(const TokenCredentials & cred std::lock_guard lock{mutex}; if (token_processors.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Token authentication is not configured"); /// lookup token in local cache if not expired. auto cached_entry_iter = access_token_to_username_cache.find(credentials.getToken()); From 8cff67f81b1fb0ecf333df860f5c85786e63b51b Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Thu, 7 Aug 2025 22:22:09 +0200 Subject: [PATCH 14/24] better docs --- .../external-authenticators/tokens.md | 117 +++++++++------- src/Access/Common/JWKSProvider.cpp | 4 +- src/Access/Common/JWKSProvider.h | 4 +- src/Access/TokenProcessorsParse.cpp | 131 ++++++++++-------- src/Parsers/Access/ParserCreateUserQuery.cpp | 10 +- 5 files changed, 156 insertions(+), 110 deletions(-) diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 5e7a20749fa1..218de276e600 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -13,26 +13,34 @@ ClickHouse users can be authenticated using tokens. This works in two ways: Although not all tokens are JWTs, under the hood both ways are treated as the same authentication method to maintain better compatibility. -## Token Processors +# Token Processors -To use token-based authentication, add `token_processors` section to `config.xml` and define at least one token processor in it. Its contents are different for different token processing workflows (JWT and opaque (generic) tokens). +## Configuration +To use token-based authentication, add `token_processors` section to `config.xml` and define at least one token processor in it. +Its contents are different for different token processor types. -### Common configuration parameters -In addition to specific parameters (see below), there are also parameters that can be applied to a token processor of any type: -- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. +**Common parameters** +- `type` -- type of token processor. Supported values: "JWT", "Azure", "OpenID". Mandatory. Case-insensitive. +- `token_cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. - `username_claim` -- name of claim (field) that will be treated as ClickHouse username. Optional, default: "sub". - `groups_claim` -- Name of claim (field) that contains list of groups user belongs to. This claim will be looked up in the token itself (in case token is a valid JWT, e.g. in Keycloak) or in response from `/userinfo`. Optional, default: "groups". -### JWT (JSON Web Token) +For each type, there are additional specific parameters. +If some parameters that are not required for current processor type are specified, they are ignored. +If there are conflicting parameters (e.g `algo` is specified together with `jwks_uri`), an exception will be thrown. -JWT itself is a source of information about user. It can be decoded locally, and its integrity can be verified using token issuer's (public) key. +## JWT (JSON Web Token) -This key can exist in three ways: -* ##### Static key: +JWT itself is a source of information about user. +It is decoded locally and its integrity is verified using either static key or JWKS (JSON Web Key Set), either local or remote. + +`algo`, `static_jwks`/`static_jwks_file` and `jwks_uri` are defining different JWT processing workflows, and they cannot be specified together. +### JWT with static key: ```xml + jwt HS256 my_static_secret @@ -49,6 +57,7 @@ This key can exist in three ways: | HS512 | RS512 | ES512 | PS512 | | | | | ES256K | | | Also supports None (though not recommended). +`claims` - A string containing a JSON object that should be contained in the token payload. If this parameter is defined, token without corresponding payload will be considered invalid. Optional. - `static_key` - key for symmetric algorithms. Mandatory for `HS*` family algorithms. - `static_key_in_base64` - indicates if the `static_key` key is base64-encoded. Optional, default: `False`. - `public_key` - public key for asymmetric algorithms. Mandatory except for `HS*` family algorithms and `None`. @@ -56,18 +65,20 @@ This key can exist in three ways: - `public_key_password` - public key password. Optional. - `private_key_password` - private key password. Optional. -* ##### Static JWKS +### JWT with static JWKS ```xml + jwt {"keys": [{"kty": "RSA", "alg": "RS256", "kid": "mykid", "n": "_public_key_mod_", "e": "AQAB"}]} ``` -#### Parameters: +**Parameters:** + - `static_jwks` - content of JWKS in JSON - `static_jwks_file` - path to a file with JWKS - `claims` - A string containing a JSON object that should be contained in the token payload. If this parameter is defined, token without corresponding payload will be considered invalid. Optional. @@ -81,63 +92,85 @@ Only one of `static_jwks` or `static_jwks_file` keys must be present in one veri Only RS* family algorithms are supported! ::: -* ##### Remote JWKS +### JWT with remote JWKS ```xml + jwt http://localhost:8000/.well-known/jwks.json - 300000 + 3600 ``` -#### Parameters: +**Parameters:** - `uri` - JWKS endpoint. Mandatory. -- `jwks_refresh_timeout` - Period for resend request for refreshing JWKS. Optional, default: 300000. +- `jwks_cache_lifetime` - Period for resend request for refreshing JWKS. Optional, default: 3600. - `claims` - A string containing a JSON object that should be contained in the token payload. If this parameter is defined, token without corresponding payload will be considered invalid. Optional. - `verifier_leeway` - Clock skew tolerance (seconds). Useful for handling small differences in system clocks between ClickHouse and the token issuer. Optional. -### Opaque tokens +## Processors with external providers + +Some tokens cannot be decoded and validated locally. External service is needed in this case. "Azure" and "OpenID" (a generic type) are supported now. -To define a token processor, add `token_processors` section to `config.xml`. Example: +### Azure ```xml - - openid - 600 - sub - groups - + + azure + ``` -:::note -Different providers have different sets of parameters. -::: +No additional parameters are required. -**Parameters** +### OpenID +```xml + + + + openid + url/.well-known/openid-configuration + 60 + 3600 + + + openid + url/userinfo + url/tokeninfo + url/.well-known/jwks.json + 60 + 3600 + + + +``` -- `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure", "OpenID". -- `configuration_endpoint` -- URI of `.well-known/openid-configuration`. Optional parameter, used only for OpenID providers. -- `userinfo_endpoint` -- URI of userinfo endpoint. Optional parameter. Optional parameter, used only for OpenID providers. -- `token_introspection_endpoint` -- URI of token introspection endpoint. Optional parameter. Optional parameter, used only for OpenID providers. - :::note -Either `configuration_endpoint` or both `userinfo_endpoint` and `token_introspection_endpoint` shall be set. If none of them are set or all three are set, this is an invalid configuration that will not be parsed. +Either `configuration_endpoint` or both `userinfo_endpoint` and `token_introspection_endpoint` (and, optionally, `jwks_uri`) shall be set. If none of them are set or all three are set, this is an invalid configuration that will not be parsed. ::: +**Parameters:** + +- `configuration_endpoint` - URI of OpenID configuration (often ends with `.well-known/openid-configuration`); +- `userinfo_endpoint` - URI of endpoint that returns user information in exchange for a valid token; +- `token_introspection_endpoint` - URI of token introspection endpoint (returns information about a valid token); +- `jwks_uri` - URI of OpenID configuration (often ends with `.well-known/jwks.json`) +- `jwks_cache_lifetime` - Period for resend request for refreshing JWKS. Optional, default: 3600. +- `verifier_leeway` - Clock skew tolerance (seconds). Useful for handling small differences in system clocks between ClickHouse and the token issuer. Optional, default: 60 +Sometimes a token is a valid JWT. In that case token will be decoded and validated locally if configuration endpoint returns JWKS URI (or `jwks_uri` is specified alongside `userinfo_endpoint` and `token_introspection_endpoint`). ### Tokens cache -To reduce number of requests to IdP, tokens are cached internally for no longer then `cache_lifetime` seconds. -If token expires sooner than `cache_lifetime`, then cache entry for this token will only be valid while token is valid. -If token lifetime is longer than `cache_lifetime`, cache entry for this token will be valid for `cache_lifetime`. +To reduce number of requests to IdP, tokens are cached internally for no longer then `token_cache_lifetime` seconds. +If token expires sooner than `token_cache_lifetime`, then cache entry for this token will only be valid while token is valid. +If token lifetime is longer than `token_cache_lifetime`, cache entry for this token will be valid for `token_cache_lifetime`. ## Enabling token authentication for a user in `users.xml` {#enabling-jwt-auth-in-users-xml} @@ -181,17 +214,7 @@ JWT authentication cannot be used together with any other authentication method. ## Enabling token authentication using SQL {#enabling-jwt-auth-using-sql} -When [SQL-driven Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled in ClickHouse, users identified by JWT authentication can also be created using SQL statements. - -```sql -CREATE USER my_user IDENTIFIED WITH jwt CLAIMS '{"resource_access":{"account": {"roles": ["view-profile"]}}}' -``` - -Or without additional JWT payload checks: - -```sql -CREATE USER my_user IDENTIFIED WITH jwt -``` +Users with "JWT" authentication type cannot be created using SQL now. ## Identity Provider as an External User Directory {#idp-external-user-directory} diff --git a/src/Access/Common/JWKSProvider.cpp b/src/Access/Common/JWKSProvider.cpp index 94ee5e04cafe..e62ba58f3121 100644 --- a/src/Access/Common/JWKSProvider.cpp +++ b/src/Access/Common/JWKSProvider.cpp @@ -19,9 +19,9 @@ jwt::jwks JWKSClient::getJWKS() std::shared_lock lock(mutex); auto now = std::chrono::high_resolution_clock::now(); - auto diff = std::chrono::duration(now - last_request_send).count(); + auto diff = std::chrono::duration(now - last_request_send).count(); - if (diff < refresh_ms) { + if (diff < refresh_timeout) { jwt::jwks result(cached_jwks); return result; } diff --git a/src/Access/Common/JWKSProvider.h b/src/Access/Common/JWKSProvider.h index 773208a138aa..45b5fcc9f4a3 100644 --- a/src/Access/Common/JWKSProvider.h +++ b/src/Access/Common/JWKSProvider.h @@ -23,7 +23,7 @@ class IJWKSProvider class JWKSClient : public IJWKSProvider { public: - explicit JWKSClient(const String & uri, const size_t refresh_ms_): refresh_ms(refresh_ms_), jwks_uri(uri) {} + explicit JWKSClient(const String & uri, const size_t refresh_ms_): refresh_timeout(refresh_ms_), jwks_uri(uri) {} ~JWKSClient() override = default; JWKSClient(const JWKSClient &) = delete; @@ -34,7 +34,7 @@ class JWKSClient : public IJWKSProvider jwt::jwks getJWKS() override; private: - size_t refresh_ms; + size_t refresh_timeout; Poco::URI jwks_uri; std::shared_mutex mutex; diff --git a/src/Access/TokenProcessorsParse.cpp b/src/Access/TokenProcessorsParse.cpp index dc0f8a16b095..59ad85ebc27f 100644 --- a/src/Access/TokenProcessorsParse.cpp +++ b/src/Access/TokenProcessorsParse.cpp @@ -15,81 +15,98 @@ std::unique_ptr ITokenProcessor::parseTokenProcessor( const String & prefix, const String & processor_name) { + if (!config.hasProperty(prefix + ".type")) + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "'type' parameter shall be specified in token_processor configuration.'"); + + auto provider_type = Poco::toLower(config.getString(prefix + ".provider")); + auto token_cache_lifetime = config.getUInt64(prefix + ".token_cache_lifetime", 3600); auto username_claim = config.getString(prefix + ".username_claim", "sub"); auto groups_claim = config.getString(prefix + ".groups_claim", "groups"); - if (config.hasProperty(prefix + ".provider")) // One of providers for opaque tokens processing + if (provider_type == "google") + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim); + } + else if (provider_type == "azure") + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim); + } + else if (provider_type == "openid") { - auto provider_type = Poco::toLower(config.getString(prefix + ".provider")); + auto verifier_leeway = config.getUInt64(prefix + ".verifier_leeway", 60); + auto jwks_cache_lifetime = config.getUInt64(prefix + ".jwks_cache_lifetime", 3600); - if (provider_type == "google") + bool externally_configured = config.hasProperty(prefix + ".configuration_endpoint") && !config.hasProperty(prefix + ".jwks_uri"); + bool locally_configured = config.hasProperty(prefix + ".userinfo_endpoint") && config.hasProperty(prefix + ".token_introspection_endpoint"); + + if (externally_configured && ! locally_configured) { - return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim); + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".openid_config_endpoint"), + verifier_leeway, + jwks_cache_lifetime); } - else if (provider_type == "azure") + else if (locally_configured && !externally_configured) { - return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim); + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".userinfo_endpoint"), + config.getString(prefix + ".token_introspection_endpoint"), + verifier_leeway, + config.getString(prefix + ".jwks_uri", ""), + jwks_cache_lifetime); } - else if (provider_type == "openid") - { - auto verifier_leeway = config.getUInt64(prefix + ".verifier_leeway", 60); - auto jwks_cache_lifetime = config.getUInt64(prefix + ".jwks_cache_lifetime", 3600); - if (config.hasProperty(prefix + ".configuration_endpoint")) - { - return std::make_unique(processor_name, token_cache_lifetime, username_claim, - groups_claim, - config.getString(prefix + ".openid_config_endpoint"), - verifier_leeway, - jwks_cache_lifetime); - } - else - { - return std::make_unique(processor_name, token_cache_lifetime, username_claim, - groups_claim, - config.getString(prefix + ".userinfo_endpoint"), - config.getString(prefix + ".token_introspection_endpoint"), - verifier_leeway, - config.getString(prefix + ".jwks_uri", ""), - jwks_cache_lifetime); - } - } + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Either 'configuration_endpoint' or both 'userinfo_endpoint' and 'token_introspection_endpoint' (and, optionally, 'jwks_uri') must be specified for 'openid' processor"); } - else if (config.hasProperty(prefix + ".algo")) /// StaticKeyJwtProcessor + else if (provider_type == "jwt") { - return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, - config.getString(prefix + ".claims", ""), - Poco::toLower(config.getString(prefix + ".algo")), - config.getString(prefix + ".static_key", ""), - config.getBool(prefix + ".static_key_in_base64", false), - config.getString(prefix + ".public_key", ""), - config.getString(prefix + ".private_key", ""), - config.getString(prefix + ".public_key_password", ""), - config.getString(prefix + ".private_key_password", "")); - } - else if (config.hasProperty(prefix + ".static_jwks") || config.hasProperty(prefix + ".static_jwks_file")) - { - StaticJWKSParams params + if (config.hasProperty(prefix + ".static_jwks") && config.hasProperty(prefix + ".static_jwks_file")) + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "'static_jwks' and 'static_jwks_file' cannot be specified simultaneously"); + + bool is_static_key = config.hasProperty(prefix + ".algo"); + bool is_static_jwks = config.hasProperty(prefix + ".static_jwks") != config.hasProperty(prefix + ".static_jwks_file"); + bool is_remote_jwks = config.hasProperty(prefix + ".jwks_uri"); + + if (is_static_key && !is_static_jwks && !is_remote_jwks) /// StaticKeyJwtProcessor { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".claims", ""), + Poco::toLower(config.getString(prefix + ".algo")), + config.getString(prefix + ".static_key", ""), + config.getBool(prefix + ".static_key_in_base64", false), + config.getString(prefix + ".public_key", ""), + config.getString(prefix + ".private_key", ""), + config.getString(prefix + ".public_key_password", ""), + config.getString(prefix + ".private_key_password", "")); + } + else if (!is_static_key && is_static_jwks && !is_remote_jwks) + { + StaticJWKSParams params + { config.getString(prefix + ".static_jwks", ""), config.getString(prefix + ".static_jwks_file", "") - }; - return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, - config.getString(prefix + ".claims", ""), - config.getUInt64(prefix + ".verifier_leeway", 0), - std::make_shared(params)); - } - else if (config.hasProperty(prefix + ".jwks_uri")) - { - return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, - config.getString(prefix + ".claims", ""), - config.getUInt64(prefix + ".verifier_leeway", 0), - config.getString(prefix + ".jwks_uri"), - config.getUInt(prefix + ".jwks_cache_lifetime", 300000)); + }; + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".claims", ""), + config.getUInt64(prefix + ".verifier_leeway", 0), + std::make_shared(params)); + } + else if (!is_static_key && !is_static_jwks && is_remote_jwks) + { + return std::make_unique(processor_name, token_cache_lifetime, username_claim, groups_claim, + config.getString(prefix + ".claims", ""), + config.getUInt64(prefix + ".verifier_leeway", 0), + config.getString(prefix + ".jwks_uri"), + config.getUInt(prefix + ".jwks_cache_lifetime", 3600)); + } + else + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "'algo', 'jwks_uri' or 'static_jwks'/'static_jwks_file' must be specified for 'jwt' processor"); } + else + throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Invalid type: {}", provider_type); - throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse token processor"); + // throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Failed to parse token processor: {}", processor_name); } diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index c1adbba0b47b..54e70d2b4573 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -25,6 +25,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace { bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, std::optional & new_name) @@ -75,7 +80,7 @@ namespace bool expect_ssl_cert_subjects = false; bool expect_public_ssh_key = false; bool expect_http_auth_server = false; - bool expect_claims = false; + bool expect_claims = false; // NOLINT auto parse_non_password_based_type = [&](auto check_type) { @@ -94,7 +99,8 @@ namespace else if (check_type == AuthenticationType::HTTP) expect_http_auth_server = true; else if (check_type == AuthenticationType::JWT) - expect_claims = true; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "CREATE USER is not supported for JWT"); + // expect_claims = true; else if (check_type != AuthenticationType::NO_PASSWORD) expect_password = true; From 1d5f0713096f0dea37b0769d95ee68d694f733ee Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Thu, 28 Aug 2025 15:04:03 +0200 Subject: [PATCH 15/24] fix typo --- src/Access/TokenProcessorsParse.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/TokenProcessorsParse.cpp b/src/Access/TokenProcessorsParse.cpp index 59ad85ebc27f..d6496db073a2 100644 --- a/src/Access/TokenProcessorsParse.cpp +++ b/src/Access/TokenProcessorsParse.cpp @@ -18,7 +18,7 @@ std::unique_ptr ITokenProcessor::parseTokenProcessor( if (!config.hasProperty(prefix + ".type")) throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "'type' parameter shall be specified in token_processor configuration.'"); - auto provider_type = Poco::toLower(config.getString(prefix + ".provider")); + auto provider_type = Poco::toLower(config.getString(prefix + ".typ")); auto token_cache_lifetime = config.getUInt64(prefix + ".token_cache_lifetime", 3600); auto username_claim = config.getString(prefix + ".username_claim", "sub"); From 4082afb3e9b77f418d7af6b20f96ad91fa03eefe Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Thu, 28 Aug 2025 20:06:46 +0200 Subject: [PATCH 16/24] fix typo --- src/Access/TokenProcessorsParse.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/TokenProcessorsParse.cpp b/src/Access/TokenProcessorsParse.cpp index d6496db073a2..4c09cd38e236 100644 --- a/src/Access/TokenProcessorsParse.cpp +++ b/src/Access/TokenProcessorsParse.cpp @@ -18,7 +18,7 @@ std::unique_ptr ITokenProcessor::parseTokenProcessor( if (!config.hasProperty(prefix + ".type")) throw DB::Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "'type' parameter shall be specified in token_processor configuration.'"); - auto provider_type = Poco::toLower(config.getString(prefix + ".typ")); + auto provider_type = Poco::toLower(config.getString(prefix + ".type")); auto token_cache_lifetime = config.getUInt64(prefix + ".token_cache_lifetime", 3600); auto username_claim = config.getString(prefix + ".username_claim", "sub"); From 0da094b89c3e19f8218c801a2e9df9e37ed4c701 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 3 Sep 2025 11:36:19 +0200 Subject: [PATCH 17/24] fix build parameter jwt --- src/configure_config.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index da044c71d130..ff6f0daf53c3 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -200,5 +200,8 @@ endif() if (TARGET ch_contrib::sha3iuf) set(USE_SHA3IUF 1) endif() +if (TARGET ch_contrib::jwt-cpp) + set(USE_JWT_CPP 1) +endif() set(SOURCE_DIR ${PROJECT_SOURCE_DIR}) From 6fa52a1f290bb48fe54b19e03288555280b1605e Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 3 Sep 2025 11:38:54 +0200 Subject: [PATCH 18/24] better logging --- src/Access/TokenProcessorsJWT.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Access/TokenProcessorsJWT.cpp b/src/Access/TokenProcessorsJWT.cpp index 170bb942f4fa..315738e9d882 100644 --- a/src/Access/TokenProcessorsJWT.cpp +++ b/src/Access/TokenProcessorsJWT.cpp @@ -324,7 +324,15 @@ bool JwksJwtProcessor::resolveAndValidate(const TokenCredentials & credentials) auto jwk = provider->getJWKS().get_jwk(decoded_jwt.get_key_id()); auto username = decoded_jwt.get_payload_claim(username_claim).as_string(); + + if (!decoded_jwt.has_algorithm()) + { + LOG_ERROR(getLogger("TokenAuthentication"), "{}: Algorithm not specified in token", processor_name); + return false; + } auto algo = Poco::toLower(decoded_jwt.get_algorithm()); + + String public_key; try From 853695c90fa8e7288aee487c38622d94b50e3515 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 3 Sep 2025 11:43:55 +0200 Subject: [PATCH 19/24] more better logging --- src/Access/TokenProcessorsJWT.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Access/TokenProcessorsJWT.cpp b/src/Access/TokenProcessorsJWT.cpp index 315738e9d882..d9ed4259b9b0 100644 --- a/src/Access/TokenProcessorsJWT.cpp +++ b/src/Access/TokenProcessorsJWT.cpp @@ -357,6 +357,8 @@ bool JwksJwtProcessor::resolveAndValidate(const TokenCredentials & credentials) if (public_key.empty()) { + if (!(jwk.has_jwk_claim("n") && jwk.has_jwk_claim("e"))) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: invalid JWK: 'n' or 'e' not found", processor_name); LOG_TRACE(getLogger("TokenAuthentication"), "{}: `issuer` or `x5c` not present, verifying {} with RSA components", processor_name, username); const auto modulus = jwk.get_jwk_claim("n").as_string(); const auto exponent = jwk.get_jwk_claim("e").as_string(); @@ -378,7 +380,7 @@ bool JwksJwtProcessor::resolveAndValidate(const TokenCredentials & credentials) verifier = verifier.leeway(verifier_leeway); verifier.verify(decoded_jwt); - if (!check_claims(claims, decoded_jwt.get_payload_json())) + if (!claims.empty() && !check_claims(claims, decoded_jwt.get_payload_json())) return false; const_cast(credentials).setUserName(decoded_jwt.get_payload_claim(username_claim).as_string()); From 4b02f55d3b955977f6853ee243a4d37237b34df2 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 3 Sep 2025 13:02:45 +0200 Subject: [PATCH 20/24] fix openid flow --- src/Access/TokenAccessStorage.cpp | 4 ++-- src/Access/TokenProcessorsJWT.cpp | 2 +- src/Access/TokenProcessorsOpaque.cpp | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index a68d89b3c3a4..831ce7d6fe12 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -351,8 +351,8 @@ std::optional TokenAccessStorage::authenticateImpl( // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. if (throw_if_user_not_exists) throwNotFound(AccessEntityType::USER, credentials.getUserName(), getStorageName()); - else - return {}; + + return {}; } std::shared_ptr new_user; diff --git a/src/Access/TokenProcessorsJWT.cpp b/src/Access/TokenProcessorsJWT.cpp index d9ed4259b9b0..dd8d3b8dd38d 100644 --- a/src/Access/TokenProcessorsJWT.cpp +++ b/src/Access/TokenProcessorsJWT.cpp @@ -388,7 +388,7 @@ bool JwksJwtProcessor::resolveAndValidate(const TokenCredentials & credentials) if (decoded_jwt.has_payload_claim(groups_claim)) const_cast(credentials).setGroups(parseGroupsFromJsonArray(decoded_jwt.get_payload_claim(groups_claim).as_array())); else - LOG_TRACE(getLogger("TokenAuthentication"), "{}: Specified groups_claim {{ not found in token, no external roles will be mapped", processor_name, groups_claim); + LOG_TRACE(getLogger("TokenAuthentication"), "{}: Specified groups_claim {} not found in token, no external roles will be mapped", processor_name, groups_claim); return true; } diff --git a/src/Access/TokenProcessorsOpaque.cpp b/src/Access/TokenProcessorsOpaque.cpp index 9a458f0d8694..095d82ffe57c 100644 --- a/src/Access/TokenProcessorsOpaque.cpp +++ b/src/Access/TokenProcessorsOpaque.cpp @@ -308,6 +308,7 @@ bool OpenIdTokenProcessor::resolveAndValidate(const TokenCredentials & credentia { auto decoded_token = jwt::decode(token); user_info_json = decoded_token.get_payload_json(); + username = getValueByKey(user_info_json, username_claim); /// TODO: Now we work only with Keycloak -- and it provides expires_at in token itself. Need to add actual token introspection logic for other OIDC providers. if (decoded_token.has_expires_at()) @@ -338,7 +339,8 @@ bool OpenIdTokenProcessor::resolveAndValidate(const TokenCredentials & credentia LOG_TRACE(getLogger("TokenAuthentication"), "{}: Failed to obtain user info", processor_name); return false; } - else if (username.empty()) + + if (username.empty()) { LOG_TRACE(getLogger("TokenAuthentication"), "{}: Failed to get username", processor_name); return false; From 4ebb534dc54ffa322a75404387ae071d02b5e301 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 3 Sep 2025 13:28:18 +0200 Subject: [PATCH 21/24] fix processing of invalid token --- src/Server/TCPHandler.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index a36ffb2d6878..5e87059f798d 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1865,7 +1865,8 @@ void TCPHandler::receiveHello() const auto & external_authenticators = server.context()->getAccessControl().getExternalAuthenticators(); - external_authenticators.checkTokenCredentials(credentials); + if (!external_authenticators.checkTokenCredentials(credentials)) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Token is invalid"); session->authenticate(credentials, getClientAddress(client_info)); return; From 1c2489d8da0697b04bb35e5b1da5c2a90f7a5c99 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Thu, 4 Sep 2025 12:57:06 +0200 Subject: [PATCH 22/24] fix role mapping --- src/Access/TokenAccessStorage.cpp | 19 +++++++++++-------- src/Access/TokenAccessStorage.h | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index 831ce7d6fe12..53af102c6827 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -20,13 +20,15 @@ namespace ErrorCodes TokenAccessStorage::TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) : IAccessStorage(storage_name_), access_control(access_control_), config(config_), prefix(prefix_), - roles_filter(config.getString(prefix.empty() ? "" : prefix + "." + "roles_filter", "")), memory_storage(storage_name_, access_control.getChangesNotifier(), false) { std::lock_guard lock(mutex); const String prefix_str = (prefix.empty() ? "" : prefix + "."); + if (config.has(prefix_str + "roles_filter")) + roles_filter.emplace(config.getString(prefix_str + "roles_filter")); + provider_name = config.getString(prefix_str + "processor"); if (provider_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "'processor' must be specified for Token user directory"); @@ -369,21 +371,22 @@ std::optional TokenAccessStorage::authenticateImpl( throwAddressNotAllowed(address); std::set external_roles; - if (!roles_filter.ok()) - { - external_roles = token_credentials.getGroups(); - LOG_TRACE(getLogger(), "{}: No external role filtering set, applying all available groups", getStorageName()); - } - else + if (roles_filter.has_value() && roles_filter.value().ok()) { + LOG_TRACE(getLogger(), "{}: External role filter found, applying only matching groups", getStorageName()); for (const auto & group: token_credentials.getGroups()) { - if (RE2::FullMatch(group, roles_filter)) + if (RE2::FullMatch(group, roles_filter.value())) { external_roles.insert(group); LOG_TRACE(getLogger(), "{}: Granted role (group) {} to user", getStorageName(), user->getName()); } } } + else + { + LOG_TRACE(getLogger(), "{}: No external role filtering set, applying all available groups", getStorageName()); + external_roles = token_credentials.getGroups(); + } if (new_user) { diff --git a/src/Access/TokenAccessStorage.h b/src/Access/TokenAccessStorage.h index d9fdf758d25f..a9414a7d0f50 100644 --- a/src/Access/TokenAccessStorage.h +++ b/src/Access/TokenAccessStorage.h @@ -48,7 +48,7 @@ class TokenAccessStorage : public IAccessStorage const String & prefix; String provider_name; - re2::RE2 roles_filter; + std::optional roles_filter = std::nullopt; std::set common_role_names; // role name that should be granted to all users at all times mutable std::map external_role_hashes; From e49ed34e9bdfc9261c89b0b39e3379b16bd1f416 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Thu, 4 Sep 2025 18:24:33 +0200 Subject: [PATCH 23/24] fix common roles parsing --- src/Access/TokenAccessStorage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index 53af102c6827..4e364d443715 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -37,7 +37,7 @@ TokenAccessStorage::TokenAccessStorage(const String & storage_name_, AccessContr if (config.has(prefix_str + "common_roles")) { Poco::Util::AbstractConfiguration::Keys role_names; - config.keys(prefix_str + "roles", role_names); + config.keys(prefix_str + "common_roles", role_names); common_roles_cfg.insert(role_names.begin(), role_names.end()); } From 134ffb7730716ba34535b66bbbdb5991f16b136a Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Mon, 22 Sep 2025 00:58:25 +0200 Subject: [PATCH 24/24] fix groups in azure --- src/Access/TokenProcessorsOpaque.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Access/TokenProcessorsOpaque.cpp b/src/Access/TokenProcessorsOpaque.cpp index 095d82ffe57c..dc1e87ee7126 100644 --- a/src/Access/TokenProcessorsOpaque.cpp +++ b/src/Access/TokenProcessorsOpaque.cpp @@ -224,7 +224,10 @@ bool AzureTokenProcessor::resolveAndValidate(const TokenCredentials & credential } auto group_data = group.get(); - String group_name = getValueByKey(group_data, "id"); + if (!group_data.contains("displayName")) + continue; + + String group_name = getValueByKey(group_data, "displayName"); external_groups_names.insert(group_name); LOG_TRACE(getLogger("TokenAuthentication"), "{}: User {}: new external group {}", processor_name, credentials.getUserName(), group_name); }