diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 49eafc6e3a1..1f234e6fdd3 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -573,6 +573,9 @@ jobs: org.apache.comet.vector.CometVectorUtilsSuite org.apache.comet.udf.CometUdfBridgeSuite org.apache.comet.objectstore.NativeConfigSuite + org.apache.comet.objectstore.AbfsAuthResolverSuite + org.apache.comet.objectstore.AbfsAuthParitySuite + org.apache.comet.objectstore.AbfsScanFallbackSuite org.apache.comet.serde.operator.CometIcebergNativeScanSuite org.apache.comet.serde.operator.CometNativeScanSuite org.apache.spark.sql.CometToPrettyStringSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 22bd6251dcf..16017718547 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -277,6 +277,9 @@ jobs: org.apache.comet.vector.CometVectorUtilsSuite org.apache.comet.udf.CometUdfBridgeSuite org.apache.comet.objectstore.NativeConfigSuite + org.apache.comet.objectstore.AbfsAuthResolverSuite + org.apache.comet.objectstore.AbfsAuthParitySuite + org.apache.comet.objectstore.AbfsScanFallbackSuite org.apache.comet.serde.operator.CometIcebergNativeScanSuite org.apache.comet.serde.operator.CometNativeScanSuite org.apache.spark.sql.CometToPrettyStringSuite diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 597a7894cd7..30e822a6e4e 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -30,5 +30,6 @@ spark/src/test/resources/tpch-query-results/*.out spark/src/test/resources/tpch-extended/q*.sql spark/src/test/resources/test-data/*.csv spark/src/test/resources/test-data/*.ndjson +spark/src/test/resources/abfs-auth-parity/*.json spark/inspections/CometTPC*results.txt benchmarks/tpc/queries/**/*.sql diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index 4bcd1834e8e..d1190124016 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -574,6 +574,38 @@ SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometT SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-4.0 -nsu test ``` +### ABFS Authentication Parity Files + +`AbfsAuthParitySuite` pins how `AbfsAuthResolver` resolves Azure ABFS authentication on each +hadoop-azure release. `spark/src/test/resources/abfs-auth-parity/expected-.json` holds, +for every case in `AbfsAuthParityCases`, the outcome kind (`resolved`, `unconfigured`, `declined`, +`failed`), the auth type, the provider class, the error class and the forwarded values, with +secrets stored as SHA-256 digests. The suite picks the file from the hadoop-azure on the test +classpath, so each Spark profile checks one version: + +| Profile | hadoop-azure | +| ------------- | ------------ | +| `-Pspark-3.5` | 3.3.4 | +| `-Pspark-4.0` | 3.4.1 | +| `-Pspark-4.1` | 3.4.2 | +| `-Pspark-4.2` | 3.5.0 | + +A new case or a hadoop-azure bump requires re-recording all four files. Run the suite once per +profile with `-Dcomet.abfs.parity.record=` and copy the written files into the resources +directory: + +```sh +./mvnw -Pspark-3.5 test -Dtest=none -Dsuites=org.apache.comet.objectstore.AbfsAuthParitySuite -Dsurefire.failIfNoSpecifiedTests=false -Dcomet.abfs.parity.record= +./mvnw -Pspark-4.0 test -Dtest=none -Dsuites=org.apache.comet.objectstore.AbfsAuthParitySuite -Dsurefire.failIfNoSpecifiedTests=false -Dcomet.abfs.parity.record= +./mvnw -Pspark-4.1 test -Dtest=none -Dsuites=org.apache.comet.objectstore.AbfsAuthParitySuite -Dsurefire.failIfNoSpecifiedTests=false -Dcomet.abfs.parity.record= +./mvnw -Pspark-4.2 test -Dtest=none -Dsuites=org.apache.comet.objectstore.AbfsAuthParitySuite -Dsurefire.failIfNoSpecifiedTests=false -Dcomet.abfs.parity.record= +``` + +The per-version pins in the cases and the cross-check against a real `AzureBlobFileSystem` +still assert while recording, so a recorded file cannot contradict what hadoop-azure does. The +file is written only when every case ran. Review the diff of the regenerated files before +committing them: each line that changes is a behaviour change on that hadoop-azure release. + ## Benchmark There's a `make` command to run micro benchmarks in the repo. For diff --git a/docs/source/user-guide/latest/datasources.md b/docs/source/user-guide/latest/datasources.md index f351aff5b6f..0a5fe4aded4 100644 --- a/docs/source/user-guide/latest/datasources.md +++ b/docs/source/user-guide/latest/datasources.md @@ -394,65 +394,50 @@ Comet's S3 support has the following limitations: Comet's Parquet scan reads Azure Data Lake Storage Gen2 (ADLS Gen2) through the [`object_store` crate](https://crates.io/crates/object_store), using the `abfs` and `abfss` URL schemes. -As with S3, standard Hadoop ABFS configurations (the `fs.azure.*` keys you already set in `core-site.xml` or via `spark.hadoop.*`) are translated into the `object_store` crate's format, so existing configurations continue to work. +The Hadoop ABFS configuration you already have (the `fs.azure.*` keys in `core-site.xml` or under `spark.hadoop.*`) authenticates the native scan as well, so an existing setup keeps working. -URLs use the same shape Spark and Hadoop emit: `abfss://@.dfs.core.windows.net/`. The account is taken from the host and the container from the URL user-info. The `wasb`, `wasbs`, `az`, `azure`, and `adl` schemes are not supported by the native scan. +URLs use the shape Spark and Hadoop emit: `abfss://@.dfs.core.windows.net/`. The `wasb`, `wasbs`, `az`, `azure`, and `adl` schemes are not supported by the native scan. ### Root CA Certificates Azure scans discover Root CA Certificates the same way S3 scans do. See [Root CA Certificates](#root-ca-certificates) above. The Rust-based scan uses system Root CA Certificates rather than the Java Trust Store. -### Supported Authentication +### Authentication is resolved by Hadoop -Comet first calls `MicrosoftAzureBuilder::from_env()`, so any `AZURE_*` environment variables (`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_FEDERATED_TOKEN_FILE`, `AZURE_AUTHORITY_HOST`, `AZURE_STORAGE_*`) are honored out of the box. This is what makes AKS Workload Identity work in a stock pod with no extra configuration. Any Hadoop `fs.azure.*` keys below are then applied on top, overriding the environment. +For each `abfs` or `abfss` path, the Spark driver asks the hadoop-azure library on its classpath which authentication mechanism applies to that container and account, and which values it needs. The question goes to the same `AbfsConfiguration` the ABFS `FileSystem` uses, so every Hadoop rule holds: account-scoped keys (`.`), container-scoped keys (`..`, hadoop-azure 3.4.2 and later), credential providers behind `hadoop.security.credential.provider.path` (a JCEKS keystore, for example) and `${...}` substitution behave exactly as they do for Spark's own reads. This holds for the hadoop-azure matching each Spark line's Hadoop: 3.3.4 with Spark 3.4 and 3.5, 3.4.1 with 4.0, 3.4.2 with 4.1, 3.5.0 with 4.2. -| Authentication method | Hadoop keys | -| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Shared account key | `fs.azure.account.key` | -| OAuth2 client credentials | `fs.azure.account.oauth2.client.id`, `fs.azure.account.oauth2.client.secret`, `fs.azure.account.oauth2.client.endpoint` (tenant id is extracted from the endpoint URL) | -| Managed identity (MSI) | `fs.azure.account.oauth2.msi.tenant`, `fs.azure.account.oauth2.msi.endpoint`, `fs.azure.account.oauth2.msi.authority` | -| Workload Identity | `fs.azure.account.oauth2.client.id`, `fs.azure.account.oauth2.msi.tenant`, `fs.azure.account.oauth2.token.file` | -| SAS token | `fs.azure.sas..` | +The native scan receives the resolved values and builds the `object_store` client from them. It reads no `fs.azure.*` key of its own. A scan whose files span more than one ABFS container or account, or that mixes `abfs`/`abfss` paths with other schemes, falls back to Spark's own scan. -### Hadoop-to-object_store key mapping +hadoop-azure must be on the driver classpath. It already is whenever Spark can list the path. When it is missing, the scan fails with an error that says so. -Internally, each Hadoop key is translated into a specific `AzureConfigKey` on the underlying `object_store` `MicrosoftAzureBuilder`: +#### Mechanisms -| Hadoop key (account-scoped suffix omitted) | `AzureConfigKey` | -| ------------------------------------------ | ------------------------ | -| `fs.azure.account.key` | `AccessKey` | -| `fs.azure.account.oauth2.client.id` | `ClientId` | -| `fs.azure.account.oauth2.client.secret` | `ClientSecret` | -| `fs.azure.account.oauth2.client.endpoint` | `AuthorityId` (from URL) | -| `fs.azure.account.oauth2.msi.tenant` | `AuthorityId` | -| `fs.azure.account.oauth2.msi.endpoint` | `MsiEndpoint` | -| `fs.azure.account.oauth2.msi.authority` | `AuthorityHost` | -| `fs.azure.account.oauth2.token.file` | `FederatedTokenFile` | -| `fs.azure.sas..` | `SasKey` | +| Hadoop mechanism | Native scan | Keys the native scan receives | +| ------------------------------------------------------------------------------------------------------------------------------ | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SharedKey` (Hadoop's default when `fs.azure.account.auth.type` is unset), with any configured `fs.azure.account.keyprovider` | Supported | `fs.azure.account.key`. Hadoop's key provider runs on the driver and the key it returns is forwarded | +| `OAuth` with `ClientCredsTokenProvider` | Supported | `fs.azure.account.oauth2.client.id`, `fs.azure.account.oauth2.client.secret`, `fs.azure.account.oauth2.client.endpoint` (the tenant and authority host come from the endpoint URL) | +| `OAuth` with `MsiTokenProvider` | Supported | `fs.azure.account.oauth2.msi.endpoint`, plus `fs.azure.account.oauth2.client.id` when set. `fs.azure.account.oauth2.msi.tenant` and `fs.azure.account.oauth2.msi.authority` are accepted but unused: `object_store`'s IMDS provider sends only the client id to the endpoint | +| `OAuth` with `WorkloadIdentityTokenProvider` (hadoop-azure 3.4.1 and later) | Supported | `fs.azure.account.oauth2.client.id`, `fs.azure.account.oauth2.msi.tenant`, `fs.azure.account.oauth2.token.file`, `fs.azure.account.oauth2.msi.authority` (the last two take Hadoop's defaults when unset) | +| `SAS` with the fixed token (hadoop-azure 3.4.1 and later) | Supported | `fs.azure.sas.fixed.token` | +| `OAuth` with `RefreshTokenBasedTokenProvider` or `UserPasswordTokenProvider` | Not supported | | +| `SAS` with a `fs.azure.sas.token.provider.type` class | Not supported | | +| `fs.azure.account.auth.type=Custom` | Not supported | | +| `WorkloadIdentityTokenProvider` with a `fs.azure.account.oauth2.client.assertion.provider.type` (hadoop-azure 3.5.0 and later) | Not supported | | +| `UserboundSASWithOAuth` (hadoop-azure 3.5.0 and later) | Not supported | | -Anything beyond these keys is not translated and falls through to whatever `from_env()` or the URL itself provided. +Custom provider classes hand each token request to Java code the native scan cannot call, and the refresh token and user password flows have no `object_store` counterpart. When Hadoop selects one of these, the scan fails with an error naming the auth type and, where one is configured, the class. Configure a supported mechanism for that account, or keep the path on Spark's own scan. Values arrive as Hadoop resolved them, trimmed and defaulted where the ABFS driver trims and defaults them, so the store sees what the driver would have used. -### Tenant id resolution +Client credentials and Workload Identity token requests go over HTTPS only. The managed identity request goes to the IMDS endpoint over HTTP, as Hadoop's does. For client credentials, Hadoop posts to `fs.azure.account.oauth2.client.endpoint` as given, while `object_store` posts to `//oauth2/v2.0/token`. Comet takes the tenant from the path segment before the last `oauth2` and everything before that segment as the authority host, so a v1 `/oauth2/token` endpoint and a proxy with a path prefix both resolve. The token request path is always the v2.0 one. An `http://` endpoint fails. -`AuthorityId` (the AAD tenant id) can be supplied in two ways: +### Environment variables -- **Directly**, via `fs.azure.account.oauth2.msi.tenant`. -- **Indirectly**, via `fs.azure.account.oauth2.client.endpoint`, which is a full token URL such as `https://login.microsoftonline.com//oauth2/token`. Comet extracts the tenant id from the first path segment of that URL. +The `AZURE_*` variables (`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_FEDERATED_TOKEN_FILE`, `AZURE_STORAGE_ACCOUNT_KEY`, `AZURE_USE_AZURE_CLI`, and the others `object_store` reads) and `IDENTITY_ENDPOINT` are read only when Hadoop configures no authentication for the account at all: none of `fs.azure.account.auth.type`, `fs.azure.account.key`, `fs.azure.account.keyprovider`, `fs.azure.shellkeyprovider.script`, `fs.azure.account.oauth.provider.type`, `fs.azure.sas.token.provider.type`, `fs.azure.sas.fixed.token` or any `fs.azure.account.oauth2.*` key resolves for it. That case is what lets AKS Workload Identity work with no Hadoop configuration for tables whose files Spark does not list through Hadoop, such as Iceberg tables. A plain Parquet path still needs Hadoop configured for Spark's own listing. -If both are set, the value from `msi.tenant` wins. +Once Hadoop names anything, only the resolved values reach the store. A resolution error (a missing mandatory key, an invalid account key, an unknown provider class) fails the scan with Hadoop's exception. Messages that could quote a credential, such as the key provider's, are kept in the Spark driver log and the scan error names the exception class. The environment is never a fallback. One read happens inside `object_store` rather than Comet: under a resolved managed identity, its IMDS provider picks up `IDENTITY_HEADER` when it fetches a token. -### Account-scoped key lookup order +### Credentials in transit -Account-scoped keys take precedence over global ones, mirroring Hadoop ABFS's own precedence. For each Hadoop key above, Comet probes the following names in order and uses the first match: - -1. `..dfs.core.windows.net` -2. `..blob.core.windows.net` -3. `.` -4. `` (unscoped / global) - -The SAS namespace follows the same shape with the container inlined into the key name: `fs.azure.sas...dfs.core.windows.net`, then `fs.azure.sas...blob.core.windows.net`, then `fs.azure.sas..`. - -The Hadoop values, when present, override anything already picked up from the `AZURE_*` environment. +The resolved values travel in the native plan from the driver to the executors, as the static `fs.s3a.access.key` and `fs.s3a.secret.key` values of an S3 configuration do. Spark's RPC channel is plaintext by default. `spark.network.crypto.enabled` or `spark.ssl.rpc.enabled` protects it. See [Wire encryption](s3-credential-providers.md#wire-encryption). ### Examples @@ -467,45 +452,65 @@ $SPARK_HOME/bin/spark-shell \ **Example 2: Workload Identity (AKS)** -In an AKS pod with Workload Identity enabled, the `AZURE_*` environment variables injected by the webhook are picked up automatically, so no Comet-specific configuration is required. To set the values explicitly instead: +In an AKS pod with Workload Identity enabled and no `fs.azure.*` authentication configured, the `AZURE_*` variables the webhook injects are picked up, so nothing Comet-specific is needed. To configure it through Hadoop instead (hadoop-azure 3.4.1 and later): ```shell $SPARK_HOME/bin/spark-shell \ ... +--conf spark.hadoop.fs.azure.account.auth.type.myaccount.dfs.core.windows.net=OAuth \ +--conf spark.hadoop.fs.azure.account.oauth.provider.type.myaccount.dfs.core.windows.net=org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider \ --conf spark.hadoop.fs.azure.account.oauth2.client.id.myaccount.dfs.core.windows.net= \ ---conf spark.hadoop.fs.azure.account.oauth2.msi.tenant.myaccount.dfs.core.windows.net= \ ---conf spark.hadoop.fs.azure.account.oauth2.token.file.myaccount.dfs.core.windows.net=/var/run/secrets/azure/tokens/azure-identity-token +--conf spark.hadoop.fs.azure.account.oauth2.msi.tenant.myaccount.dfs.core.windows.net= ... ``` -**Example 3: OAuth2 client credentials with tenant embedded in the endpoint URL** +The token file defaults to `/var/run/secrets/azure/tokens/azure-identity-token`, the path AKS mounts. Set `fs.azure.account.oauth2.token.file` for another location. -If only `fs.azure.account.oauth2.client.endpoint` is set, the tenant id is parsed from the endpoint path automatically — there is no need to set `msi.tenant` separately. +**Example 3: OAuth2 client credentials** + +The tenant comes from the endpoint URL, so `fs.azure.account.oauth2.msi.tenant` is not needed: ```shell $SPARK_HOME/bin/spark-shell \ ... +--conf spark.hadoop.fs.azure.account.auth.type.myaccount.dfs.core.windows.net=OAuth \ +--conf spark.hadoop.fs.azure.account.oauth.provider.type.myaccount.dfs.core.windows.net=org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider \ --conf spark.hadoop.fs.azure.account.oauth2.client.id.myaccount.dfs.core.windows.net= \ --conf spark.hadoop.fs.azure.account.oauth2.client.secret.myaccount.dfs.core.windows.net= \ --conf spark.hadoop.fs.azure.account.oauth2.client.endpoint.myaccount.dfs.core.windows.net=https://login.microsoftonline.com//oauth2/token ... ``` -**Example 4: SAS token scoped to a container** +**Example 4: SAS token** + +Hadoop's ABFS driver (3.4.1 and later) reads the fixed token under `fs.azure.account.auth.type=SAS`, and the native scan receives the same value, so one configuration serves the driver and the executors: ```shell $SPARK_HOME/bin/spark-shell \ ... ---conf spark.hadoop.fs.azure.sas.mycontainer.myaccount.dfs.core.windows.net='sv=2020-08-04&sig=...' +--conf spark.hadoop.fs.azure.account.auth.type.myaccount.dfs.core.windows.net=SAS \ +--conf spark.hadoop.fs.azure.sas.fixed.token.myaccount.dfs.core.windows.net='sv=2020-08-04&sig=...' ... ``` +To give one container its own token on hadoop-azure 3.4.2 and later, set `fs.azure.sas.fixed.token.mycontainer.myaccount.dfs.core.windows.net`. Hadoop reads it before the account-level key, and the native scan receives whichever one Hadoop picked. The OAuth keys take the same container-scoped form on those versions. + +### Changes from Comet 1.1 + +Comet 1.1 started from the environment and overlaid the Hadoop keys it found by probing the account name under two endpoint suffixes and bare. Now Hadoop decides. What this changes: + +- Container-scoped keys work on hadoop-azure 3.4.2 and later, as they do for Spark. +- Transport variables such as `AZURE_STORAGE_ENDPOINT` and `AZURE_ALLOW_HTTP` no longer apply beside a Hadoop mechanism. They are read only when Hadoop configures none. +- The WASB key `fs.azure.sas..`, which the ABFS driver never read, is no longer read by the native scan either. Use `fs.azure.sas.fixed.token`, which the native scan now reads. +- A lower-case `oauth` auth type, or OAuth keys without `fs.azure.account.auth.type`, fail the way they fail for Spark. In the second case Hadoop defaults to `SharedKey` and the key provider fails. On hadoop-azure 3.4.1 and later the driver log names the missing key. 3.3.4 reports only `Failure to initialize configuration`. +- A configured mechanism the native scan cannot build fails with an error instead of silently reading the environment. + ### Limitations -1. **Partial Hadoop ABFS configuration support**: Only the `fs.azure.*` keys listed above are translated and applied to the underlying `object_store` crate. +1. **Authentication keys only**: Only the keys in the table above reach the native store. Other `fs.azure.*` settings (read-ahead, retries, HTTP tuning) do not apply to the native scan. 2. **Supported schemes**: Only `abfs` and `abfss` are routed to the native Azure store. `wasb[s]`, `az`, `azure`, and `adl` are not supported. `wasb[s]` is not recognised by `object_store` at all; `az`, `azure`, and `adl` are recognised by `object_store` but treat the URL host as the _container_ rather than the _account_, which is incompatible with Hadoop's account-scoped configuration keys. 3. **URL shape**: URLs must include the account in the host, i.e. `abfss://@.dfs.core.windows.net/`. Bare `abfs:///` (fsspec-style, no account in the URL) is not supported because Comet cannot resolve the storage account name. -4. **Endpoint suffixes for account-scoped lookup**: Comet probes for account-scoped keys under `dfs.core.windows.net` and `blob.core.windows.net`. Custom or sovereign-cloud endpoint suffixes (e.g. `dfs.core.chinacloudapi.cn`, Fabric endpoints) are not probed; use the unscoped `.` form for those. +4. **Endpoint hosts**: `object_store` builds an Azure store only for hosts under `dfs.core.windows.net`, `blob.core.windows.net`, `dfs.fabric.microsoft.com` and `blob.fabric.microsoft.com`. Sovereign-cloud endpoints such as `dfs.core.chinacloudapi.cn` are not supported, whatever keys are configured for them. diff --git a/native/core/src/parquet/objectstore/azure.rs b/native/core/src/parquet/objectstore/azure.rs index 6fed99f1642..4b49cc07eb3 100644 --- a/native/core/src/parquet/objectstore/azure.rs +++ b/native/core/src/parquet/objectstore/azure.rs @@ -15,54 +15,63 @@ // specific language governing permissions and limitations // under the License. -//! Construct a `MicrosoftAzure` object store from a Hadoop ABFS configuration. +//! Construct a `MicrosoftAzure` object store from the ABFS authentication the driver resolved. //! -//! Comet's native scans run outside the JVM, so they bypass Hadoop's -//! `AzureBlobFileSystem` driver entirely. This module bridges the gap by translating the -//! Hadoop `fs.azure.*` configuration namespace (the same keys users already put in -//! `core-site.xml` or `spark.hadoop.*`) into the `object_store` crate's `AzureConfigKey` -//! options, then layering them on top of any AZURE_* environment variables that AKS's -//! Workload Identity webhook injects. +//! Comet's native scans run outside the JVM, so they cannot use Hadoop's `AzureBlobFileSystem`. +//! Which mechanism applies to an `abfs[s]://@/` URL, and which values +//! it needs, is decided on the Spark driver by `AbfsAuthResolver`, which asks the classpath's +//! hadoop-azure the same questions `AzureBlobFileSystemStore.initializeClient` asks. Account-, +//! container- and globally-scoped keys, credential providers and `${...}` references are all +//! Hadoop's business there. This module trusts that answer and only maps it onto +//! `MicrosoftAzureBuilder`. The answer arrives in the object store options as `comet.azure.*` +//! markers plus, for a resolved mechanism, the values under their global Hadoop key names. //! -//! Only `abfs[s]://@./` URLs are supported — -//! the same URL shape Spark and Hadoop emit. `wasb[s]://` is not routed here because -//! `object_store::ObjectStoreScheme::parse` does not recognise it (it only accepts -//! `az | adl | azure | abfs | abfss`). The `az` / `azure` / `adl` schemes are also out of -//! scope: `object_store`'s `MicrosoftAzureBuilder::parse_url` treats the URL host as the -//! *container* for those schemes rather than the *account*, which does not match the -//! account-scoped Hadoop key layout that `NativeConfig` forwards. +//! `comet.azure.resolution` decides everything: //! -//! Supported authentication, in the priority order applied to the builder: +//! - `resolved`: `MicrosoftAzureBuilder::new()` plus exactly the keys the mechanism owns. +//! - `none`: `MicrosoftAzureBuilder::from_env()`, the only path in Comet's own code that reads +//! `AZURE_*` or `IDENTITY_ENDPOINT`. (object_store's IMDS provider still reads +//! `IDENTITY_HEADER` at token-fetch time under a resolved managed identity; that is its +//! behaviour, not a Comet lookup.) +//! - `declined`: an error naming the auth type and the provider class Hadoop configured. +//! - `error`: an error carrying the driver's exception class and message. +//! - missing or anything else: an error, so a plan that did not run the resolver cannot pass. //! -//! 1. `MicrosoftAzureBuilder::from_env()` — picks up `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, -//! `AZURE_FEDERATED_TOKEN_FILE`, `AZURE_AUTHORITY_HOST`, `AZURE_STORAGE_*`, etc. -//! This is what makes Workload Identity work out of the box in AKS pods. -//! 2. Account-scoped Hadoop keys (`fs.azure.account.X..dfs.core.windows.net`). -//! 3. Global Hadoop keys (`fs.azure.account.X`). +//! `comet.azure.auth.type` is Hadoop's `AuthType` name, `comet.azure.oauth.provider.class` the +//! OAuth provider class, `comet.azure.sas.provider.class` a custom SAS provider and +//! `comet.azure.oauth.assertion.provider.class` a Workload Identity client assertion provider. +//! Under `resolved` each mechanism maps the keys its Hadoop provider constructor reads: //! -//! Items 2 and 3 are forwarded via `MicrosoftAzureBuilder::with_config`, which overrides -//! whatever `from_env()` produced. The account-scoped variant wins over the global one, -//! mirroring Hadoop ABFS's own `AbfsConfiguration` precedence. +//! | Mechanism | Hadoop key (global form) | `AzureConfigKey` | +//! | --- | --- | --- | +//! | `SharedKey` | `fs.azure.account.key` | `AccessKey` | +//! | `ClientCredsTokenProvider` | `fs.azure.account.oauth2.client.id` | `ClientId` | +//! | | `fs.azure.account.oauth2.client.secret` | `ClientSecret` | +//! | | `fs.azure.account.oauth2.client.endpoint` | `AuthorityId`, `AuthorityHost` | +//! | `MsiTokenProvider` | `fs.azure.account.oauth2.msi.endpoint` | `MsiEndpoint` | +//! | | `fs.azure.account.oauth2.client.id` | `ClientId` (optional) | +//! | | `fs.azure.account.oauth2.msi.tenant` | `AuthorityId` (optional) | +//! | | `fs.azure.account.oauth2.msi.authority` | `AuthorityHost` (optional) | +//! | `WorkloadIdentityTokenProvider` | `fs.azure.account.oauth2.client.id` | `ClientId` | +//! | | `fs.azure.account.oauth2.msi.tenant` | `AuthorityId` | +//! | | `fs.azure.account.oauth2.token.file` | `FederatedTokenFile` | +//! | | `fs.azure.account.oauth2.msi.authority` | `AuthorityHost` | +//! | `SAS` (fixed token) | `fs.azure.sas.fixed.token` | `SasKey` | //! -//! The translated keys cover the auth schemes that ABFS users actually configure: -//! -//! | Hadoop key (account-scoped suffix omitted) | `AzureConfigKey` | -//! | ------------------------------------------------------- | ---------------------- | -//! | `fs.azure.account.key` | `AccessKey` | -//! | `fs.azure.account.oauth2.client.id` | `ClientId` | -//! | `fs.azure.account.oauth2.client.secret` | `ClientSecret` | -//! | `fs.azure.account.oauth2.client.endpoint` | `AuthorityId` (from URL) | -//! | `fs.azure.account.oauth2.msi.tenant` | `AuthorityId` | -//! | `fs.azure.account.oauth2.msi.endpoint` | `MsiEndpoint` | -//! | `fs.azure.account.oauth2.msi.authority` | `AuthorityHost` | -//! | `fs.azure.account.oauth2.token.file` | `FederatedTokenFile` | -//! | `fs.azure.sas..` | `SasKey` | -//! -//! Anything beyond these falls through to whatever `from_env()` or the URL itself provided. +//! An optional key that is absent or blank counts as unset, as Hadoop's `MsiTokenProvider` adds +//! the client id and tenant to its request only when they are non-empty. The MSI tenant and +//! authority are accepted but unused: object_store's IMDS provider reads only the client id and +//! endpoint. `RefreshTokenBasedTokenProvider` and `UserPasswordTokenProvider` have no +//! `object_store` counterpart and fail as unsupported. A value a mechanism needs but that is +//! absent (or, for the SAS token and the client credentials endpoint, unusable) is an error +//! naming the key; other `fs.azure.*` keys are ignored. Values are applied as forwarded (Hadoop +//! already trimmed and defaulted them), except that the trailing slash Hadoop appends to +//! `msi.authority` is removed because `object_store` joins `//...` +//! itself. No error message or log line carries a value, only key names. use log::debug; use std::collections::HashMap; -use url::Url; +use url::{Position, Url}; use object_store::{ azure::{AzureConfigKey, MicrosoftAzureBuilder}, @@ -70,6 +79,27 @@ use object_store::{ ObjectStore, ObjectStoreScheme, }; +const MARKER_RESOLUTION: &str = "comet.azure.resolution"; +const MARKER_AUTH_TYPE: &str = "comet.azure.auth.type"; +const MARKER_OAUTH_PROVIDER_CLASS: &str = "comet.azure.oauth.provider.class"; +const MARKER_OAUTH_ASSERTION_PROVIDER_CLASS: &str = "comet.azure.oauth.assertion.provider.class"; +const MARKER_SAS_PROVIDER_CLASS: &str = "comet.azure.sas.provider.class"; +const MARKER_ERROR_CLASS: &str = "comet.azure.error.class"; +const MARKER_ERROR_MESSAGE: &str = "comet.azure.error.message"; + +const AUTH_TYPE_SHARED_KEY: &str = "SharedKey"; +const AUTH_TYPE_OAUTH: &str = "OAuth"; +const AUTH_TYPE_SAS: &str = "SAS"; + +const PROVIDER_CLIENT_CREDS: &str = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"; +const PROVIDER_MSI: &str = "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider"; +const PROVIDER_WORKLOAD_IDENTITY: &str = + "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider"; +const PROVIDER_REFRESH_TOKEN: &str = + "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider"; +const PROVIDER_USER_PASSWORD: &str = + "org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider"; + const HADOOP_KEY: &str = "fs.azure.account.key"; const HADOOP_OAUTH_CLIENT_ID: &str = "fs.azure.account.oauth2.client.id"; const HADOOP_OAUTH_CLIENT_SECRET: &str = "fs.azure.account.oauth2.client.secret"; @@ -78,11 +108,11 @@ const HADOOP_MSI_TENANT: &str = "fs.azure.account.oauth2.msi.tenant"; const HADOOP_MSI_ENDPOINT: &str = "fs.azure.account.oauth2.msi.endpoint"; const HADOOP_MSI_AUTHORITY: &str = "fs.azure.account.oauth2.msi.authority"; const HADOOP_WI_TOKEN_FILE: &str = "fs.azure.account.oauth2.token.file"; -const HADOOP_SAS_PREFIX: &str = "fs.azure.sas."; +const HADOOP_SAS_FIXED_TOKEN: &str = "fs.azure.sas.fixed.token"; -const ENDPOINT_SUFFIXES: &[&str] = &["dfs.core.windows.net", "blob.core.windows.net"]; +type Settings = Vec<(AzureConfigKey, String)>; -/// Build a `MicrosoftAzure` `ObjectStore` for `url` using `configs`. +/// Build a `MicrosoftAzure` `ObjectStore` for `url` from the driver's resolved authentication. /// /// `url` must use the `abfs[s]://` scheme; the returned `Path` is the URL's resource path /// (container-relative), suitable for direct use with `ObjectStore::get`. @@ -92,338 +122,879 @@ pub fn create_store( ) -> Result<(Box, Path), object_store::Error> { let (scheme, path) = ObjectStoreScheme::parse(url)?; if scheme != ObjectStoreScheme::MicrosoftAzure { - return Err(object_store::Error::Generic { - store: "MicrosoftAzure", - source: format!("Scheme of URL is not Azure: {url}").into(), - }); + return Err(config_error(format!("Scheme of URL is not Azure: {url}"))); } let path = Path::parse(path)?; - - let account = extract_account(url); - let container = extract_container(url); - - // Start from the environment so AKS Workload Identity (AZURE_CLIENT_ID, - // AZURE_TENANT_ID, AZURE_FEDERATED_TOKEN_FILE, AZURE_AUTHORITY_HOST) and any - // explicit AZURE_STORAGE_* variables are honoured without further configuration. - // `with_url` then fills in account/container from the URL itself. - let mut builder = MicrosoftAzureBuilder::from_env().with_url(url.to_string()); - - let translated = translate_hadoop_configs(configs, account.as_deref(), container.as_deref()); - debug!( - "Azure configs for account={:?}, container={:?}: keys={:?}", - account, - container, - translated - .iter() - .map(|(k, _)| k.as_ref()) - .collect::>() - ); - for (key, value) in translated { - builder = builder.with_config(key, value); - } - - let store = builder.build()?; + let store = builder_for(url, configs)?.build()?; Ok((Box::new(store), path)) } -/// Translate a Hadoop ABFS configuration map into `(AzureConfigKey, value)` pairs. -/// -/// `account` and `container` are extracted from the URL and used to resolve account-scoped -/// keys (`fs.azure.X..`) and the SAS namespace -/// (`fs.azure.sas..`). Account-scoped keys win over global ones. -fn translate_hadoop_configs( +/// The builder the markers in `configs` call for. Only `none` touches the environment. +fn builder_for( + url: &Url, configs: &HashMap, - account: Option<&str>, - container: Option<&str>, -) -> Vec<(AzureConfigKey, String)> { - let mut out: Vec<(AzureConfigKey, String)> = Vec::new(); - - let mappings: &[(&str, AzureConfigKey)] = &[ - (HADOOP_KEY, AzureConfigKey::AccessKey), - (HADOOP_OAUTH_CLIENT_ID, AzureConfigKey::ClientId), - (HADOOP_OAUTH_CLIENT_SECRET, AzureConfigKey::ClientSecret), - (HADOOP_MSI_TENANT, AzureConfigKey::AuthorityId), - (HADOOP_MSI_ENDPOINT, AzureConfigKey::MsiEndpoint), - (HADOOP_MSI_AUTHORITY, AzureConfigKey::AuthorityHost), - (HADOOP_WI_TOKEN_FILE, AzureConfigKey::FederatedTokenFile), - ]; - - for (hadoop_base, azure_key) in mappings { - if let Some(value) = account_scoped_value(configs, hadoop_base, account) { - out.push((*azure_key, value)); +) -> Result { + let marker = |key: &str| configs.get(key).map(String::as_str); + match marker(MARKER_RESOLUTION) { + Some("none") => { + debug!("Azure authentication for {url}: none configured in Hadoop, reading AZURE_*"); + Ok(MicrosoftAzureBuilder::from_env().with_url(url.to_string())) + } + Some("resolved") => { + let settings = resolved_settings(configs)?; + debug!( + "Azure authentication for {url}: {} with keys {:?}", + marker(MARKER_AUTH_TYPE).unwrap_or_default(), + settings.iter().map(|(k, _)| k.as_ref()).collect::>() + ); + let builder = MicrosoftAzureBuilder::new().with_url(url.to_string()); + Ok(settings + .into_iter() + .fold(builder, |b, (key, value)| b.with_config(key, value))) } + Some("declined") => { + let class = marker(MARKER_OAUTH_ASSERTION_PROVIDER_CLASS) + .or_else(|| marker(MARKER_SAS_PROVIDER_CLASS)) + .or_else(|| marker(MARKER_OAUTH_PROVIDER_CLASS)); + Err(unsupported( + marker(MARKER_AUTH_TYPE).unwrap_or(""), + class, + )) + } + Some("error") => Err(config_error(format!( + "Hadoop ABFS authentication failed on the driver for {}://{}: {}: {}", + url.scheme(), + &url[Position::BeforeUsername..Position::AfterPort], + marker(MARKER_ERROR_CLASS).unwrap_or(""), + marker(MARKER_ERROR_MESSAGE).unwrap_or(""), + ))), + Some(_) => Err(config_error(format!( + "unknown `{MARKER_RESOLUTION}` value; the native ABFS scan needs the driver's Hadoop \ + resolution" + ))), + None => Err(config_error(format!( + "`{MARKER_RESOLUTION}` is missing; the native ABFS scan needs the driver's Hadoop \ + resolution" + ))), } +} - // `fs.azure.account.oauth2.client.endpoint` is a full token URL of the form - // `https://login.microsoftonline.com//oauth2/token`. object_store wants the - // tenant id directly (`AuthorityId`), so extract it if AuthorityId hasn't already - // been set from `fs.azure.account.oauth2.msi.tenant`. - let has_authority_id = out - .iter() - .any(|(k, _)| matches!(k, AzureConfigKey::AuthorityId)); - if !has_authority_id { - if let Some(endpoint) = account_scoped_value(configs, HADOOP_OAUTH_CLIENT_ENDPOINT, account) - { - if let Some(tenant) = tenant_from_oauth_endpoint(&endpoint) { - out.push((AzureConfigKey::AuthorityId, tenant)); +/// The builder settings for the mechanism the auth type and provider class markers name: the +/// keys its Hadoop provider constructor reads, and nothing else. +fn resolved_settings(configs: &HashMap) -> Result { + let marker = |key: &str| configs.get(key).map(String::as_str); + let auth_type = marker(MARKER_AUTH_TYPE) + .ok_or_else(|| config_error(format!("`{MARKER_AUTH_TYPE}` is missing")))?; + if let Some(class) = + marker(MARKER_OAUTH_ASSERTION_PROVIDER_CLASS).or_else(|| marker(MARKER_SAS_PROVIDER_CLASS)) + { + return Err(unsupported(auth_type, Some(class))); + } + match (auth_type, marker(MARKER_OAUTH_PROVIDER_CLASS)) { + (AUTH_TYPE_SHARED_KEY, _) => Ok(vec![( + AzureConfigKey::AccessKey, + required(configs, HADOOP_KEY)?, + )]), + (AUTH_TYPE_SAS, _) => { + let token = required(configs, HADOOP_SAS_FIXED_TOKEN)?; + if token.is_empty() { + return Err(config_error(format!("`{HADOOP_SAS_FIXED_TOKEN}` is empty"))); } + Ok(vec![(AzureConfigKey::SasKey, token)]) } - } - - // SAS tokens are scoped to `fs.azure.sas..[.]`. - if let (Some(container), Some(account)) = (container, account) { - if let Some(sas) = sas_value(configs, container, account) { - out.push((AzureConfigKey::SasKey, sas)); + (AUTH_TYPE_OAUTH, Some(PROVIDER_CLIENT_CREDS)) => client_creds_settings(configs), + (AUTH_TYPE_OAUTH, Some(PROVIDER_MSI)) => msi_settings(configs), + (AUTH_TYPE_OAUTH, Some(PROVIDER_WORKLOAD_IDENTITY)) => workload_identity_settings(configs), + (AUTH_TYPE_OAUTH, Some(class @ (PROVIDER_REFRESH_TOKEN | PROVIDER_USER_PASSWORD))) => { + Err(unsupported(auth_type, Some(class))) } + (AUTH_TYPE_OAUTH, Some(_)) => Err(config_error(format!( + "unknown `{MARKER_OAUTH_PROVIDER_CLASS}` for the native ABFS scan" + ))), + (AUTH_TYPE_OAUTH, None) => Err(config_error(format!( + "`{MARKER_OAUTH_PROVIDER_CLASS}` is missing" + ))), + _ => Err(config_error(format!( + "unknown `{MARKER_AUTH_TYPE}` for the native ABFS scan" + ))), } +} + +/// `ClientCredsTokenProvider` posts to the endpoint itself, so the endpoint supplies both the +/// tenant and the authority host; an endpoint without either is an error, not a default. +fn client_creds_settings( + configs: &HashMap, +) -> Result { + let endpoint = required(configs, HADOOP_OAUTH_CLIENT_ENDPOINT)?; + let (tenant, host) = oauth_endpoint_parts(&endpoint).ok_or_else(|| { + config_error(format!( + "`{HADOOP_OAUTH_CLIENT_ENDPOINT}` has no `scheme://host` origin or no tenant segment \ + before `oauth2`" + )) + })?; + Ok(vec![ + ( + AzureConfigKey::ClientId, + required(configs, HADOOP_OAUTH_CLIENT_ID)?, + ), + ( + AzureConfigKey::ClientSecret, + required(configs, HADOOP_OAUTH_CLIENT_SECRET)?, + ), + (AzureConfigKey::AuthorityId, tenant), + (AzureConfigKey::AuthorityHost, host), + ]) +} - out +/// `MsiTokenProvider`: the endpoint is required (Hadoop defaults it); the identity is optional. +fn msi_settings(configs: &HashMap) -> Result { + let mut out = vec![( + AzureConfigKey::MsiEndpoint, + required(configs, HADOOP_MSI_ENDPOINT)?, + )]; + out.extend(optional( + configs, + HADOOP_OAUTH_CLIENT_ID, + AzureConfigKey::ClientId, + )); + out.extend(optional( + configs, + HADOOP_MSI_TENANT, + AzureConfigKey::AuthorityId, + )); + out.extend(optional( + configs, + HADOOP_MSI_AUTHORITY, + AzureConfigKey::AuthorityHost, + )); + Ok(out) } -/// Look up `base_key`, preferring account-scoped variants. -/// -/// Probes (in order): `..`, `.`, -/// then the unscoped ``. Returns the first hit. -fn account_scoped_value( +/// `WorkloadIdentityTokenProvider`: every key is mandatory or defaulted in Hadoop. +fn workload_identity_settings( configs: &HashMap, - base_key: &str, - account: Option<&str>, -) -> Option { - if let Some(acc) = account { - for suffix in ENDPOINT_SUFFIXES { - let scoped = format!("{base_key}.{acc}.{suffix}"); - if let Some(v) = configs.get(&scoped) { - return Some(v.clone()); - } - } - let bare = format!("{base_key}.{acc}"); - if let Some(v) = configs.get(&bare) { - return Some(v.clone()); - } - } - configs.get(base_key).cloned() +) -> Result { + Ok(vec![ + ( + AzureConfigKey::ClientId, + required(configs, HADOOP_OAUTH_CLIENT_ID)?, + ), + ( + AzureConfigKey::AuthorityId, + required(configs, HADOOP_MSI_TENANT)?, + ), + ( + AzureConfigKey::FederatedTokenFile, + required(configs, HADOOP_WI_TOKEN_FILE)?, + ), + ( + AzureConfigKey::AuthorityHost, + authority_host(&required(configs, HADOOP_MSI_AUTHORITY)?), + ), + ]) } -/// Resolve the SAS token for `(container, account)`, accepting any of the -/// `fs.azure.sas..[.]` variants. -fn sas_value(configs: &HashMap, container: &str, account: &str) -> Option { - for suffix in ENDPOINT_SUFFIXES { - let key = format!("{HADOOP_SAS_PREFIX}{container}.{account}.{suffix}"); - if let Some(v) = configs.get(&key) { - return Some(v.clone()); - } - } - let bare = format!("{HADOOP_SAS_PREFIX}{container}.{account}"); - configs.get(&bare).cloned() +fn required(configs: &HashMap, key: &str) -> Result { + configs.get(key).cloned().ok_or_else(|| { + config_error(format!( + "`{key}` is missing from the resolved ABFS authentication" + )) + }) } -/// Extract the storage account name from an `abfs[s]://` URL. -/// -/// ABFS hostnames are `.` (e.g. `myacct.dfs.core.windows.net`), -/// so the account is the first label of the host. -fn extract_account(url: &Url) -> Option { - let host = url.host_str()?; - host.split('.').next().map(str::to_string) +/// An optional key, where blank counts as unset (`MsiTokenProvider` sends a client id or tenant +/// only when it is non-empty, untrimmed). The authority loses Hadoop's trailing slash. +fn optional( + configs: &HashMap, + key: &str, + azure_key: AzureConfigKey, +) -> Option<(AzureConfigKey, String)> { + let value = configs.get(key).filter(|v| !v.is_empty())?; + let value = match azure_key { + AzureConfigKey::AuthorityHost => authority_host(value), + _ => value.clone(), + }; + Some((azure_key, value)) } -/// Extract the container name from an `abfs[s]://` URL. -/// -/// ABFS encodes the container as the URL user-info (`container@account.dfs...`). -fn extract_container(url: &Url) -> Option { - let user = url.username(); - if user.is_empty() { - return None; +/// Hadoop appends a slash; object_store joins `//oauth2/v2.0/token`. +fn authority_host(authority: &str) -> String { + authority.trim_end_matches('/').to_string() +} + +fn config_error(message: String) -> object_store::Error { + object_store::Error::Generic { + store: "MicrosoftAzure", + source: message.into(), } - Some(user.to_string()) } -/// Pull the tenant id out of an OAuth token endpoint like -/// `https://login.microsoftonline.com//oauth2/token`. -fn tenant_from_oauth_endpoint(endpoint: &str) -> Option { +fn unsupported(auth_type: &str, class: Option<&str>) -> object_store::Error { + let provider = class + .map(|c| format!(" with provider {c}")) + .unwrap_or_default(); + config_error(format!( + "unsupported authentication mechanism for the native ABFS scan: auth type \ + {auth_type}{provider}" + )) +} + +/// The tenant id and authority host of an OAuth token endpoint like +/// `https://login.microsoftonline.com//oauth2/token`. The tenant is the path segment +/// before the last `oauth2`, or the first segment when no later segment is `oauth2`, so a +/// prefix that itself contains `oauth2` is kept. The authority host is the origin plus every +/// segment before the tenant, so object_store's `//oauth2/v2.0/token` +/// keeps a proxy's path prefix. `None` when the endpoint has no tenant or no `scheme://host` +/// origin. +fn oauth_endpoint_parts(endpoint: &str) -> Option<(String, String)> { let parsed = Url::parse(endpoint).ok()?; - let mut segments = parsed.path_segments()?; - let tenant = segments.next()?; - if tenant.is_empty() { + let segments: Vec<&str> = parsed.path_segments()?.collect(); + let tenant_at = segments + .iter() + .rposition(|segment| *segment == "oauth2") + .filter(|&at| at > 0) + .map_or(0, |at| at - 1); + let tenant = segments.get(tenant_at).filter(|t| !t.is_empty())?; + let origin = parsed.origin(); + if !origin.is_tuple() { return None; } - Some(tenant.to_string()) + let mut host = origin.ascii_serialization(); + for segment in &segments[..tenant_at] { + host.push('/'); + host.push_str(segment); + } + Some((tenant.to_string(), host)) } #[cfg(test)] mod tests { use super::*; + use object_store::client::{ + HttpClient, HttpConnector, HttpError, HttpRequest, HttpResponse, HttpResponseBody, + HttpService, + }; + use object_store::ClientOptions; + use std::sync::Mutex; + + const URL: &str = "abfss://data@myacct.dfs.core.windows.net/path/file.parquet"; + /// Marks every credential value; no error may echo it. + const SECRET: &str = "SECRET-VALUE-MARKER"; + /// A valid base64 account key, since `build()` decodes it. + const KEY: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + const DEFAULT_ENDPOINT: &str = "https://login.microsoftonline.com/hadoop-tenant/oauth2/token"; + const DEFAULT_MSI_ENDPOINT: &str = "http://169.254.169.254/metadata/identity/oauth2/token"; + const DEFAULT_AUTHORITY: &str = "https://login.microsoftonline.com/"; + const PUBLIC_CLOUD: &str = "https://login.microsoftonline.com"; + const TOKEN_FILE: &str = "/var/run/secrets/azure/tokens/azure-identity-token"; + const ENV_KEY: &str = "AZURE_STORAGE_ACCOUNT_KEY"; + const ENV_CLIENT_ID: &str = "AZURE_CLIENT_ID"; + const UNSUPPORTED: &str = "unsupported authentication mechanism"; + + /// Serializes the tests that mutate the process environment. + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + fn lock_env() -> std::sync::MutexGuard<'static, ()> { + ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()) + } - fn url(s: &str) -> Url { - Url::parse(s).unwrap() + fn url() -> Url { + Url::parse(URL).unwrap() } - #[test] - fn extracts_account_and_container_from_abfss_url() { - let u = url("abfss://data@myacct.dfs.core.windows.net/path/file.parquet"); - assert_eq!(extract_account(&u).as_deref(), Some("myacct")); - assert_eq!(extract_container(&u).as_deref(), Some("data")); + type Configs = HashMap; + + fn configs(pairs: &[(&str, &str)]) -> Configs { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + fn with(mut configs: Configs, key: &str, value: &str) -> Configs { + configs.insert(key.into(), value.into()); + configs + } + + fn without(mut configs: Configs, key: &str) -> Configs { + configs.remove(key); + configs + } + + fn resolved(auth_type: &str, pairs: &[(&str, &str)]) -> Configs { + let out = with(configs(pairs), MARKER_RESOLUTION, "resolved"); + with(out, MARKER_AUTH_TYPE, auth_type) + } + + fn shared_key() -> Configs { + resolved(AUTH_TYPE_SHARED_KEY, &[(HADOOP_KEY, SECRET)]) + } + + fn oauth(provider: &str, pairs: &[(&str, &str)]) -> Configs { + with( + resolved(AUTH_TYPE_OAUTH, pairs), + MARKER_OAUTH_PROVIDER_CLASS, + provider, + ) + } + + fn client_creds() -> Configs { + oauth( + PROVIDER_CLIENT_CREDS, + &[ + (HADOOP_OAUTH_CLIENT_ID, "hadoop-client"), + (HADOOP_OAUTH_CLIENT_SECRET, SECRET), + (HADOOP_OAUTH_CLIENT_ENDPOINT, DEFAULT_ENDPOINT), + ], + ) + } + + fn msi() -> Configs { + oauth( + PROVIDER_MSI, + &[ + (HADOOP_MSI_ENDPOINT, DEFAULT_MSI_ENDPOINT), + (HADOOP_MSI_AUTHORITY, DEFAULT_AUTHORITY), + ], + ) + } + + fn workload_identity() -> Configs { + oauth( + PROVIDER_WORKLOAD_IDENTITY, + &[ + (HADOOP_OAUTH_CLIENT_ID, "hadoop-client"), + (HADOOP_MSI_TENANT, "hadoop-tenant"), + (HADOOP_WI_TOKEN_FILE, TOKEN_FILE), + (HADOOP_MSI_AUTHORITY, DEFAULT_AUTHORITY), + ], + ) + } + + fn builder(configs: &Configs) -> MicrosoftAzureBuilder { + builder_for(&url(), configs).expect("builder") + } + + fn value(builder: &MicrosoftAzureBuilder, key: AzureConfigKey) -> Option { + builder.get_config_value(&key) + } + + /// Every credential-bearing key the builder could receive, for "nothing else was set". + const CREDENTIAL_KEYS: &[AzureConfigKey] = &[ + AzureConfigKey::AccessKey, + AzureConfigKey::SasKey, + AzureConfigKey::ClientId, + AzureConfigKey::ClientSecret, + AzureConfigKey::AuthorityId, + AzureConfigKey::AuthorityHost, + AzureConfigKey::MsiEndpoint, + AzureConfigKey::FederatedTokenFile, + AzureConfigKey::Token, + ]; + + fn assert_exactly(configs: &Configs, expected: &[(AzureConfigKey, &str)]) { + let builder = builder(configs); + for key in CREDENTIAL_KEYS { + let want = expected.iter().find(|(k, _)| k == key).map(|(_, v)| *v); + assert_eq!(value(&builder, *key).as_deref(), want, "{key:?}"); + } + } + + /// The error `configs` produce, which must not echo the secret marker. + fn err(configs: &Configs) -> String { + let err = builder_for(&url(), configs) + .err() + .map(|e| e.to_string()) + .expect("an error"); + assert!(!err.contains(SECRET), "value echoed: {err}"); + err } #[test] - fn account_scoped_key_takes_precedence_over_global() { - let mut configs = HashMap::new(); - configs.insert( - "fs.azure.account.oauth2.client.id".into(), - "global-client".into(), - ); - configs.insert( - "fs.azure.account.oauth2.client.id.myacct.dfs.core.windows.net".into(), - "scoped-client".into(), - ); - assert_eq!( - account_scoped_value(&configs, HADOOP_OAUTH_CLIENT_ID, Some("myacct")).as_deref(), - Some("scoped-client"), - ); + fn shared_key_sets_only_the_access_key_and_ignores_other_hadoop_keys() { + assert_exactly(&shared_key(), &[(AzureConfigKey::AccessKey, SECRET)]); + let stray = with(shared_key(), HADOOP_SAS_FIXED_TOKEN, "sv=1&sig=x"); + let stray = with(stray, HADOOP_OAUTH_CLIENT_ID, "stray"); + let stray = with(stray, "fs.azure.account.hns.enabled", "true"); + assert_exactly(&stray, &[(AzureConfigKey::AccessKey, SECRET)]); } #[test] - fn account_scoped_lookup_falls_back_to_global() { - let mut configs = HashMap::new(); - configs.insert( - "fs.azure.account.oauth2.client.id".into(), - "global-client".into(), - ); - assert_eq!( - account_scoped_value(&configs, HADOOP_OAUTH_CLIENT_ID, Some("myacct")).as_deref(), - Some("global-client"), - ); + fn create_store_checks_the_scheme_and_returns_the_container_path() { + let configs = with(shared_key(), HADOOP_KEY, KEY); + let (_store, path) = create_store(&url(), &configs).expect("store builds"); + assert_eq!(path.as_ref(), "path/file.parquet"); + let s3 = Url::parse("s3://bucket/file.parquet").unwrap(); + let err = create_store(&s3, &configs).unwrap_err().to_string(); + assert!(err.contains("Scheme of URL is not Azure"), "{err}"); } #[test] - fn translates_workload_identity_keys() { - let mut configs = HashMap::new(); - configs.insert( - "fs.azure.account.oauth2.client.id.myacct.dfs.core.windows.net".into(), - "client-123".into(), + fn client_creds_take_tenant_and_authority_host_from_the_endpoint_alone() { + assert_exactly( + &client_creds(), + &[ + (AzureConfigKey::ClientId, "hadoop-client"), + (AzureConfigKey::ClientSecret, SECRET), + (AzureConfigKey::AuthorityId, "hadoop-tenant"), + (AzureConfigKey::AuthorityHost, PUBLIC_CLOUD), + ], ); - configs.insert( - "fs.azure.account.oauth2.msi.tenant.myacct.dfs.core.windows.net".into(), - "tenant-abc".into(), + // `msi.*` keys from a shared configuration are not read under this provider. + let proxied = with(client_creds(), HADOOP_MSI_TENANT, "other-tenant"); + let proxied = with( + proxied, + HADOOP_MSI_AUTHORITY, + "https://login.chinacloudapi.cn/", ); - configs.insert( - "fs.azure.account.oauth2.token.file.myacct.dfs.core.windows.net".into(), - "/var/run/secrets/azure/tokens/azure-identity-token".into(), - ); - let translated = translate_hadoop_configs(&configs, Some("myacct"), Some("data")); - - let by_key: HashMap<_, _> = translated.into_iter().collect(); - assert_eq!( - by_key.get(&AzureConfigKey::ClientId).map(String::as_str), - Some("client-123") + let proxied = with( + proxied, + HADOOP_OAUTH_CLIENT_ENDPOINT, + "https://auth-proxy.example:8443/aad/hadoop-tenant/oauth2/v2.0/token", ); + let builder = builder(&proxied); assert_eq!( - by_key.get(&AzureConfigKey::AuthorityId).map(String::as_str), - Some("tenant-abc") + value(&builder, AzureConfigKey::AuthorityId).as_deref(), + Some("hadoop-tenant") ); assert_eq!( - by_key - .get(&AzureConfigKey::FederatedTokenFile) - .map(String::as_str), - Some("/var/run/secrets/azure/tokens/azure-identity-token") + value(&builder, AzureConfigKey::AuthorityHost).as_deref(), + Some("https://auth-proxy.example:8443/aad") ); + // An endpoint without a tenant segment or without a `scheme://host` origin cannot + // fall through to another credential or to the default authority host. + for endpoint in [ + "https://auth-proxy.example/", + "file:///hadoop-tenant/oauth2/token", + ] { + let err = err(&with( + client_creds(), + HADOOP_OAUTH_CLIENT_ENDPOINT, + endpoint, + )); + assert!( + err.contains(HADOOP_OAUTH_CLIENT_ENDPOINT), + "{endpoint}: {err}" + ); + } + } + + /// Records the request URLs a store sends so the token request is observed offline. + #[derive(Debug, Default, Clone)] + struct RecordingConnector(std::sync::Arc>>); + + #[async_trait::async_trait] + impl HttpService for RecordingConnector { + async fn call(&self, req: HttpRequest) -> Result { + self.0.lock().unwrap().push(req.uri().to_string()); + Ok(http::Response::builder() + .status(400) + .body(HttpResponseBody::from(String::new())) + .unwrap()) + } + } + + impl HttpConnector for RecordingConnector { + fn connect(&self, _options: &ClientOptions) -> object_store::Result { + Ok(HttpClient::new(self.clone())) + } + } + + #[tokio::test] + async fn client_creds_token_requests_go_to_the_endpoint_host() { + // Hadoop posts to `client.endpoint` itself, so a proxy mounted under a path keeps that + // path in the authority host and the tenant is the segment before the last `oauth2`. + for (endpoint, token_url) in [ + ( + DEFAULT_ENDPOINT, + "https://login.microsoftonline.com/hadoop-tenant/oauth2/v2.0/token", + ), + ( + "https://auth-proxy.example/gateway/oauth2/hadoop-tenant/oauth2/v2.0/token", + "https://auth-proxy.example/gateway/oauth2/hadoop-tenant/oauth2/v2.0/token", + ), + ] { + let configs = with(client_creds(), HADOOP_OAUTH_CLIENT_ENDPOINT, endpoint); + let recorder = RecordingConnector::default(); + let store = builder(&configs) + .with_http_connector(recorder.clone()) + .build() + .expect("store builds"); + assert!(store.credentials().get_credential().await.is_err()); + let urls = recorder.0.lock().unwrap(); + assert_eq!( + urls.first().map(String::as_str), + Some(token_url), + "{endpoint}" + ); + } } #[test] - fn translates_account_key() { - let mut configs = HashMap::new(); - configs.insert( - "fs.azure.account.key.myacct.blob.core.windows.net".into(), - "secret==".into(), + fn msi_sets_the_endpoint_with_an_optional_identity_and_authority() { + assert_exactly( + &msi(), + &[ + (AzureConfigKey::MsiEndpoint, DEFAULT_MSI_ENDPOINT), + (AzureConfigKey::AuthorityHost, PUBLIC_CLOUD), + ], ); - let translated = translate_hadoop_configs(&configs, Some("myacct"), None); - let by_key: HashMap<_, _> = translated.into_iter().collect(); - assert_eq!( - by_key.get(&AzureConfigKey::AccessKey).map(String::as_str), - Some("secret==") + let user_assigned = with(msi(), HADOOP_OAUTH_CLIENT_ID, "user-assigned"); + let user_assigned = with(user_assigned, HADOOP_MSI_TENANT, "hadoop-tenant"); + assert_exactly( + &user_assigned, + &[ + (AzureConfigKey::MsiEndpoint, DEFAULT_MSI_ENDPOINT), + (AzureConfigKey::AuthorityHost, PUBLIC_CLOUD), + (AzureConfigKey::ClientId, "user-assigned"), + (AzureConfigKey::AuthorityId, "hadoop-tenant"), + ], + ); + // Hadoop forwards a blank client id or tenant as "" and sends neither; the authority is + // defaulted in Hadoop but unused by object_store's IMDS provider, so it may be absent. + let blank = with(msi(), HADOOP_OAUTH_CLIENT_ID, ""); + let blank = with(blank, HADOOP_MSI_TENANT, ""); + let blank = without(blank, HADOOP_MSI_AUTHORITY); + assert_exactly( + &blank, + &[(AzureConfigKey::MsiEndpoint, DEFAULT_MSI_ENDPOINT)], ); } #[test] - fn translates_sas_token_for_container() { - let mut configs = HashMap::new(); - configs.insert( - "fs.azure.sas.data.myacct.dfs.core.windows.net".into(), - "sv=2020-08-04&sig=xyz".into(), - ); - let translated = translate_hadoop_configs(&configs, Some("myacct"), Some("data")); - let by_key: HashMap<_, _> = translated.into_iter().collect(); - assert_eq!( - by_key.get(&AzureConfigKey::SasKey).map(String::as_str), - Some("sv=2020-08-04&sig=xyz") + fn workload_identity_sets_client_tenant_token_file_and_authority() { + assert_exactly( + &workload_identity(), + &[ + (AzureConfigKey::ClientId, "hadoop-client"), + (AzureConfigKey::AuthorityId, "hadoop-tenant"), + (AzureConfigKey::FederatedTokenFile, TOKEN_FILE), + (AzureConfigKey::AuthorityHost, PUBLIC_CLOUD), + ], ); + // Hadoop appends a slash to the authority; object_store joins `//...`. + for (authority, host) in [ + ( + "https://login.chinacloudapi.cn/", + "https://login.chinacloudapi.cn", + ), + ( + "https://login.chinacloudapi.cn", + "https://login.chinacloudapi.cn", + ), + ( + "https://auth-proxy.example/aad/", + "https://auth-proxy.example/aad", + ), + ] { + let configs = with(workload_identity(), HADOOP_MSI_AUTHORITY, authority); + assert_eq!( + value(&builder(&configs), AzureConfigKey::AuthorityHost).as_deref(), + Some(host), + "{authority}" + ); + } } #[test] - fn derives_tenant_from_oauth_endpoint_when_msi_tenant_is_absent() { - let mut configs = HashMap::new(); - configs.insert( - "fs.azure.account.oauth2.client.endpoint".into(), - "https://login.microsoftonline.com/00000000-1111-2222-3333-444444444444/oauth2/token" - .into(), - ); - let translated = translate_hadoop_configs(&configs, Some("myacct"), None); - let by_key: HashMap<_, _> = translated.into_iter().collect(); - assert_eq!( - by_key.get(&AzureConfigKey::AuthorityId).map(String::as_str), - Some("00000000-1111-2222-3333-444444444444") - ); + fn fixed_sas_token_is_forwarded_verbatim() { + for token in [ + "sv=2020-08-04&sig=xyz", + "?sv=2020-08-04&sig=xyz", + " sv=1&sig=x ", + ] { + let configs = resolved(AUTH_TYPE_SAS, &[(HADOOP_SAS_FIXED_TOKEN, token)]); + assert_exactly(&configs, &[(AzureConfigKey::SasKey, token)]); + // `split_sas` trims the leading `?`, so the store builds from either spelling. + create_store(&url(), &configs).unwrap_or_else(|e| panic!("{token:?}: {e}")); + } + // An empty token would make object_store send unsigned requests. + let err = err(&resolved(AUTH_TYPE_SAS, &[(HADOOP_SAS_FIXED_TOKEN, "")])); + assert!(err.contains(HADOOP_SAS_FIXED_TOKEN), "{err}"); } #[test] - fn msi_tenant_wins_over_oauth_endpoint_tenant() { - let mut configs = HashMap::new(); - configs.insert( - "fs.azure.account.oauth2.msi.tenant".into(), - "from-msi".into(), - ); - configs.insert( - "fs.azure.account.oauth2.client.endpoint".into(), - "https://login.microsoftonline.com/from-endpoint/oauth2/token".into(), - ); - let translated = translate_hadoop_configs(&configs, Some("myacct"), None); - let by_key: HashMap<_, _> = translated.into_iter().collect(); + fn unsupported_mechanisms_fail_naming_the_auth_type_and_class() { + let custom = "com.example.CustomProvider"; + // (resolution, auth type, marker carrying the class, class) + let cases = [ + // Declined on the driver, where the class was never instantiated either. + ("declined", "OAuth", MARKER_OAUTH_PROVIDER_CLASS, custom), + ( + "declined", + "OAuth", + MARKER_OAUTH_ASSERTION_PROVIDER_CLASS, + custom, + ), + ("declined", "SAS", MARKER_SAS_PROVIDER_CLASS, custom), + ("declined", "Custom", "", ""), + ("declined", "UserboundSASWithOAuth", "", ""), + // Resolved by Hadoop to a provider without an object_store counterpart. + ( + "resolved", + "OAuth", + MARKER_OAUTH_PROVIDER_CLASS, + PROVIDER_REFRESH_TOKEN, + ), + ( + "resolved", + "OAuth", + MARKER_OAUTH_PROVIDER_CLASS, + PROVIDER_USER_PASSWORD, + ), + ]; + for (resolution, auth_type, marker, class) in cases { + let mut configs = configs(&[ + (MARKER_RESOLUTION, resolution), + (MARKER_AUTH_TYPE, auth_type), + (HADOOP_OAUTH_CLIENT_ID, SECRET), + ]); + if marker == MARKER_OAUTH_ASSERTION_PROVIDER_CLASS { + // The assertion provider rides beside the Workload Identity class. + configs = with( + configs, + MARKER_OAUTH_PROVIDER_CLASS, + PROVIDER_WORKLOAD_IDENTITY, + ); + } + if !marker.is_empty() { + configs = with(configs, marker, class); + } + let err = err(&configs); + assert!(err.contains(UNSUPPORTED), "{err}"); + assert!(err.contains(auth_type) && err.contains(class), "{err}"); + } + } + + #[test] + fn bad_or_missing_markers_fail_closed_naming_the_marker() { + let cases = [ + (configs(&[(HADOOP_KEY, SECRET)]), MARKER_RESOLUTION), + ( + with(shared_key(), MARKER_RESOLUTION, "Resolved"), + MARKER_RESOLUTION, + ), + (without(shared_key(), MARKER_AUTH_TYPE), MARKER_AUTH_TYPE), + ( + resolved("Kerberos", &[(HADOOP_KEY, SECRET)]), + MARKER_AUTH_TYPE, + ), + (resolved(AUTH_TYPE_OAUTH, &[]), MARKER_OAUTH_PROVIDER_CLASS), + ( + oauth("com.example.MyTokenProvider", &[]), + MARKER_OAUTH_PROVIDER_CLASS, + ), + ]; + for (configs, marker) in cases { + let err = err(&configs); + assert!(err.contains(marker), "{marker}: {err}"); + } + } + + #[test] + fn a_required_key_missing_under_resolved_fails_naming_it() { + let cases: Vec<(Configs, &[&str])> = vec![ + (shared_key(), &[HADOOP_KEY]), + ( + resolved(AUTH_TYPE_SAS, &[(HADOOP_SAS_FIXED_TOKEN, SECRET)]), + &[HADOOP_SAS_FIXED_TOKEN], + ), + ( + client_creds(), + &[ + HADOOP_OAUTH_CLIENT_ID, + HADOOP_OAUTH_CLIENT_SECRET, + HADOOP_OAUTH_CLIENT_ENDPOINT, + ], + ), + (msi(), &[HADOOP_MSI_ENDPOINT]), + ( + workload_identity(), + &[ + HADOOP_OAUTH_CLIENT_ID, + HADOOP_MSI_TENANT, + HADOOP_WI_TOKEN_FILE, + HADOOP_MSI_AUTHORITY, + ], + ), + ]; + for (complete, required) in cases { + builder(&complete); + for key in required { + let err = err(&without(complete.clone(), key)); + assert!(err.contains(key), "{key}: {err}"); + } + } + } + + #[test] + fn error_marker_surfaces_the_driver_failure() { + let configs = configs(&[ + (MARKER_RESOLUTION, "error"), + ( + MARKER_ERROR_CLASS, + "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + ), + (MARKER_ERROR_MESSAGE, "Failure to initialize configuration"), + ]); assert_eq!( - by_key.get(&AzureConfigKey::AuthorityId).map(String::as_str), - Some("from-msi") + err(&configs), + "Generic MicrosoftAzure error: Hadoop ABFS authentication failed on the driver for \ + abfss://data@myacct.dfs.core.windows.net: \ + org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException: \ + Failure to initialize configuration" ); } #[test] - fn create_store_succeeds_with_workload_identity_configs() { - let u = url("abfss://data@myacct.dfs.core.windows.net/path/file.parquet"); - let mut configs = HashMap::new(); - // A typical Workload Identity setup: client id, tenant id, and federated token - // file are all present, and `from_env()` will additionally pick them up from the - // AZURE_* env vars in production. - configs.insert( - "fs.azure.account.oauth2.client.id".into(), - "client-123".into(), - ); - configs.insert( - "fs.azure.account.oauth2.msi.tenant".into(), - "tenant-abc".into(), + fn only_none_reads_the_environment() { + let _guard = lock_env(); + std::env::set_var(ENV_KEY, KEY); + std::env::set_var(ENV_CLIENT_ID, "ambient-client"); + let from_env = builder_for(&url(), &configs(&[(MARKER_RESOLUTION, "none")])); + // Stray Hadoop values beside `none` are not read either. + let from_env_with_stray = builder_for( + &url(), + &configs(&[(MARKER_RESOLUTION, "none"), (HADOOP_KEY, SECRET)]), ); - configs.insert( - "fs.azure.account.oauth2.token.file".into(), - "/var/run/secrets/azure/tokens/azure-identity-token".into(), - ); - let (_store, path) = create_store(&u, &configs).expect("store builds"); - assert_eq!(path.as_ref(), "path/file.parquet"); + let sas = resolved(AUTH_TYPE_SAS, &[(HADOOP_SAS_FIXED_TOKEN, "sv=1&sig=x")]); + let mechanisms = [ + ("shared_key", shared_key(), Some(SECRET), None), + ("client_creds", client_creds(), None, Some("hadoop-client")), + ("msi", msi(), None, None), + ( + "workload_identity", + workload_identity(), + None, + Some("hadoop-client"), + ), + ("sas", sas, None, None), + ]; + let resolved: Vec<_> = mechanisms + .iter() + .map(|(name, configs, key, client)| { + (*name, builder_for(&url(), configs), *key, *client) + }) + .collect(); + let failing = [ + configs(&[ + (MARKER_RESOLUTION, "declined"), + (MARKER_AUTH_TYPE, "Custom"), + ]), + configs(&[(MARKER_RESOLUTION, "error")]), + configs(&[(HADOOP_KEY, SECRET)]), + ]; + let results: Vec> = failing + .iter() + .map(|configs| create_store(&url(), configs).map(|_| ())) + .collect(); + std::env::remove_var(ENV_KEY); + std::env::remove_var(ENV_CLIENT_ID); + + for builder in [from_env, from_env_with_stray] { + let builder = builder.expect("builder"); + assert_eq!( + value(&builder, AzureConfigKey::AccessKey).as_deref(), + Some(KEY) + ); + assert_eq!( + value(&builder, AzureConfigKey::ClientId).as_deref(), + Some("ambient-client") + ); + } + for (name, builder, key, client) in resolved { + let builder = builder.expect(name); + assert_eq!( + value(&builder, AzureConfigKey::AccessKey).as_deref(), + key, + "{name}" + ); + assert_eq!( + value(&builder, AzureConfigKey::ClientId).as_deref(), + client, + "{name}" + ); + } + for (configs, result) in failing.iter().zip(results) { + let err = result.err().map(|e| e.to_string()).expect("an error"); + assert!(!err.contains(KEY), "{configs:?}: {err}"); + } } #[test] - fn create_store_rejects_non_azure_scheme() { - let u = url("s3://bucket/file.parquet"); - let configs = HashMap::new(); - let err = create_store(&u, &configs).expect_err("must fail"); - assert!( - format!("{err}").contains("Scheme of URL is not Azure"), - "unexpected error: {err}" - ); + fn oauth_endpoint_parts_split_tenant_and_authority_host() { + fn parts(tenant: &'static str, host: &'static str) -> Option<(&'static str, &'static str)> { + Some((tenant, host)) + } + for (endpoint, expected) in [ + ( + "https://user:pass@auth-proxy.example/t/oauth2/token", + parts("t", "https://auth-proxy.example"), + ), + ( + "https://auth-proxy.example:443/t/oauth2/token", + parts("t", "https://auth-proxy.example"), + ), + ( + "https://auth-proxy.example:8443/t/oauth2/token", + parts("t", "https://auth-proxy.example:8443"), + ), + ( + "https://[::1]:8443/t/oauth2/token", + parts("t", "https://[::1]:8443"), + ), + ( + "https://auth-proxy.example/aad/t/oauth2/v2.0/token", + parts("t", "https://auth-proxy.example/aad"), + ), + ( + "https://auth-proxy.example:8443/a/b/t/oauth2/v2.0/token", + parts("t", "https://auth-proxy.example:8443/a/b"), + ), + // No `oauth2` after the first segment: the first segment is the tenant. + ( + "https://auth-proxy.example/aad/t/token", + parts("aad", "https://auth-proxy.example"), + ), + ( + "https://auth-proxy.example/oauth2/token", + parts("oauth2", "https://auth-proxy.example"), + ), + ("https://h//t/oauth2/token", parts("t", "https://h/")), + ("https://h/a//oauth2/token", None), + ( + "https://h/aad/t/oauth2/v2.0/token?q=oauth2#f", + parts("t", "https://h/aad"), + ), + ( + "https://h/a/oauth2/b/oauth2/token", + parts("b", "https://h/a/oauth2"), + ), + ( + "https://h/u@evil.example/t/oauth2/token", + parts("t", "https://h/u@evil.example"), + ), + // The URL parser resolves `%2e%2e` and reads `\` as `/`; the host stays `h`. + ("https://h/a/%2e%2e/t/oauth2/token", parts("t", "https://h")), + ("https://h/a\\b/t/oauth2/token", parts("t", "https://h/a/b")), + ( + "https://h/oauth2/t/oauth2/token", + parts("t", "https://h/oauth2"), + ), + // An opaque origin has no host to post to. + ("file:///t/oauth2/token", None), + ("https://auth-proxy.example/", None), + ("urn:t:oauth2", None), + ("not a url", None), + ] { + let actual = oauth_endpoint_parts(endpoint); + assert_eq!( + actual + .as_ref() + .map(|(tenant, host)| (tenant.as_str(), host.as_str())), + expected, + "{endpoint}" + ); + } } } diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index cc17429a72b..5765e99e944 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -1339,7 +1339,11 @@ mod tests { fn isolates_azure_containers_in_cache_and_shared_runtime() { use datafusion::execution::runtime_env::RuntimeEnvBuilder; - let options = HashMap::from([("fs.azure.account.key".into(), "c2VjcmV0".into())]); + let options = HashMap::from([ + ("comet.azure.resolution".into(), "resolved".into()), + ("comet.azure.auth.type".into(), "SharedKey".into()), + ("fs.azure.account.key".into(), "c2VjcmV0".into()), + ]); let runtime = RuntimeEnvBuilder::new() .with_object_store_registry(Arc::new(super::CometObjectStoreRegistry::default())) .build_arc() diff --git a/pom.xml b/pom.xml index a020f0719f9..714779c6d60 100644 --- a/pom.xml +++ b/pom.xml @@ -105,6 +105,11 @@ under the License. Hadoop 3.4.x's CredentialProviderListFactory); the spark-3.4 / spark-3.5 profiles override it down to 3.3.x (SDK v1). hadoop.version itself (hadoop-client) is not changed. --> 3.4.1 + + 3.4.2 ${project.basedir}/../native/target/debug darwin x86_64 @@ -705,6 +710,7 @@ under the License. 3.4.3 3.4 3.3.4 + 3.3.4 1.13.1 4.8.8 2.0.6 @@ -725,6 +731,7 @@ under the License. 3.5.9 3.5 3.3.4 + 3.3.4 1.13.1 4.8.8 2.0.7 @@ -744,6 +751,7 @@ under the License. 2.13 4.0.4 4.0 + 3.4.1 1.15.2 4.13.10 2.0.16 @@ -766,6 +774,7 @@ under the License. 2.13 4.1.3 4.1 + 3.4.2 1.16.0 4.13.10 2.0.17 @@ -786,6 +795,7 @@ under the License. 2.13 4.2.0 4.2 + 3.5.0 1.17.0 4.13.10 2.0.17 @@ -1023,6 +1033,9 @@ under the License. ${comet.test.udfs.lib} + + ${comet.abfs.parity.record} diff --git a/spark/pom.xml b/spark/pom.xml index 18c0ac60914..0dfd7bb259d 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -224,6 +224,39 @@ under the License. + + + org.apache.hadoop + hadoop-azure + ${hadoop-azure.version} + test + + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + + + org.apache.hadoop + hadoop-annotations + + + org.eclipse.jetty + jetty-util-ajax + + + commons-logging + commons-logging + + + software.amazon.awssdk diff --git a/spark/src/main/scala/org/apache/comet/objectstore/AbfsAuthResolver.scala b/spark/src/main/scala/org/apache/comet/objectstore/AbfsAuthResolver.scala new file mode 100644 index 00000000000..a7938ce79a8 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/objectstore/AbfsAuthResolver.scala @@ -0,0 +1,517 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.objectstore + +import java.net.URI + +import scala.collection.mutable.ArrayBuffer +import scala.util.{Failure, Success, Try} +import scala.util.control.NonFatal + +import org.apache.commons.lang3.StringUtils +import org.apache.hadoop.conf.Configuration +import org.apache.spark.internal.Logging + +import org.apache.comet.objectstore.AbfsReflection.Handles + +/** + * Resolves the ABFS authentication for one `abfs[s]://container@account/...` URI on the driver, + * by asking the classpath's hadoop-azure the questions + * `AzureBlobFileSystemStore.initializeClient` asks, and encodes the answer as the options the + * native Azure store reads. + * + * Hadoop's `AbfsConfiguration` decides which mechanism applies and where each value comes from + * (container-, account- or globally-scoped keys, credential providers, `${...}` references). The + * resolver forwards the resolved values under the global key names plus `comet.azure.*` markers, + * never a key for another account, and never the environment. Anything Hadoop throws becomes an + * error marker: the native scan fails with it instead of trying a lookup of its own. + * + * `AbfsAuthParitySuite` pins the outcome of every configuration shape per hadoop-azure version + * (`spark/src/test/resources/abfs-auth-parity/expected-.json`) and cross-checks it + * against the real `AzureBlobFileSystem`; its scaladoc says how to regenerate those files after a + * deliberate change here. + */ +private[comet] object AbfsAuthResolver extends Logging { + + /** Options the native Azure store reads. None of them starts with `fs.`. */ + object Markers { + val RESOLUTION = "comet.azure.resolution" + val AUTH_TYPE = "comet.azure.auth.type" + val OAUTH_PROVIDER_CLASS = "comet.azure.oauth.provider.class" + val SAS_PROVIDER_CLASS = "comet.azure.sas.provider.class" + val OAUTH_ASSERTION_PROVIDER_CLASS = "comet.azure.oauth.assertion.provider.class" + val ERROR_CLASS = "comet.azure.error.class" + val ERROR_MESSAGE = "comet.azure.error.message" + + val RESOLVED = "resolved" + val NONE = "none" + val DECLINED = "declined" + val ERROR = "error" + } + + /** Hadoop ABFS configuration keys, global form (`ConfigurationKeys`). */ + object Keys { + val AUTH_TYPE = "fs.azure.account.auth.type" + val ACCOUNT_KEY = "fs.azure.account.key" + val KEY_PROVIDER = "fs.azure.account.keyprovider" + val SHELL_KEY_PROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script" + val OAUTH_PROVIDER_TYPE = "fs.azure.account.oauth.provider.type" + val CLIENT_ID = "fs.azure.account.oauth2.client.id" + val CLIENT_SECRET = "fs.azure.account.oauth2.client.secret" + val CLIENT_ENDPOINT = "fs.azure.account.oauth2.client.endpoint" + val MSI_TENANT = "fs.azure.account.oauth2.msi.tenant" + val MSI_ENDPOINT = "fs.azure.account.oauth2.msi.endpoint" + val MSI_AUTHORITY = "fs.azure.account.oauth2.msi.authority" + val USER_NAME = "fs.azure.account.oauth2.user.name" + val USER_PASSWORD = "fs.azure.account.oauth2.user.password" + val REFRESH_TOKEN = "fs.azure.account.oauth2.refresh.token" + val REFRESH_TOKEN_ENDPOINT = "fs.azure.account.oauth2.refresh.token.endpoint" + val TOKEN_FILE = "fs.azure.account.oauth2.token.file" + val CLIENT_ASSERTION_PROVIDER_TYPE = "fs.azure.account.oauth2.client.assertion.provider.type" + val SAS_TOKEN_PROVIDER_TYPE = "fs.azure.sas.token.provider.type" + val SAS_FIXED_TOKEN = "fs.azure.sas.fixed.token" + } + + /** The OAuth providers Hadoop accepts under `AuthType.OAuth`, matched by exact class name. */ + object Providers { + val CLIENT_CREDS = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider" + val USER_PASSWORD = "org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider" + val MSI = "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider" + val REFRESH_TOKEN = "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider" + val WORKLOAD_IDENTITY = "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider" + val BUILT_IN: Set[String] = + Set(CLIENT_CREDS, USER_PASSWORD, MSI, REFRESH_TOKEN, WORKLOAD_IDENTITY) + } + + private object AuthTypes { + val SHARED_KEY = "SharedKey" + val OAUTH = "OAuth" + val SAS = "SAS" + } + + private object Exceptions { + private val pkg = "org.apache.hadoop.fs.azurebfs.contracts.exceptions." + val CONFIGURATION_PROPERTY_NOT_FOUND: String = pkg + "ConfigurationPropertyNotFoundException" + val INVALID_CONFIGURATION_VALUE: String = pkg + "InvalidConfigurationValueException" + val INVALID_URI: String = pkg + "InvalidUriException" + val INVALID_URI_AUTHORITY: String = pkg + "InvalidUriAuthorityException" + } + + /** + * Exceptions whose messages name keys, classes or URIs and nothing else, so they are forwarded + * verbatim. Every other message (a `KeyProviderException` quotes the raw account key, custom + * providers say anything) is replaced by a pointer to the driver log. + */ + private val verbatimMessageClasses: Set[String] = Set( + Exceptions.CONFIGURATION_PROPERTY_NOT_FOUND, + Exceptions.INVALID_CONFIGURATION_VALUE, + Exceptions.INVALID_URI, + Exceptions.INVALID_URI_AUTHORITY, + classOf[ClassNotFoundException].getName, + classOf[NoSuchMethodException].getName, + classOf[NoSuchFieldException].getName) + + // An IllegalArgumentException is verbatim only when Hadoop or the JDK enum parser raised it + // ("No enum constant", "Failed to initialize", "Invalid account key."). One thrown + // by a user KeyProvider could carry anything, so it is reported by class only. + private val HADOOP_AZURE_PACKAGE = "org.apache.hadoop.fs.azurebfs." + private val JAVA_ENUM_CLASS = "java.lang.Enum" + + private def isVerbatim(t: Throwable): Boolean = t match { + case _: IllegalArgumentException => + t.getStackTrace.headOption.map(_.getClassName).exists { origin => + origin == JAVA_ENUM_CLASS || origin.startsWith(HADOOP_AZURE_PACKAGE) + } + case _ => verbatimMessageClasses.contains(t.getClass.getName) + } + + private val REDACTED = "" + // Shorter values (`true`, a port) would mangle messages without protecting anything. + private val MIN_REDACTED_LENGTH = 8 + // `SimpleKeyProvider` (3.4.1+) puts the raw account key in its message as `key ="..."`. + private val QUOTED_KEY_PATTERN = "key =\"[^\"]*\"".r + private val MAX_CAUSE_DEPTH = 8 + + // The keys that name a mechanism or carry a credential in hadoop-azure. A configuration where + // none of them resolves for the account is `Unconfigured`; any one of them hands the decision to + // Hadoop, which defaults to SharedKey and fails in its own words when that does not fit. + private[objectstore] val plainAuthKeys: Seq[String] = Seq( + Keys.AUTH_TYPE, + Keys.KEY_PROVIDER, + Keys.SHELL_KEY_PROVIDER_SCRIPT, + Keys.OAUTH_PROVIDER_TYPE, + Keys.SAS_TOKEN_PROVIDER_TYPE) + private[objectstore] val passwordAuthKeys: Seq[String] = Seq( + Keys.ACCOUNT_KEY, + Keys.CLIENT_ID, + Keys.CLIENT_SECRET, + Keys.CLIENT_ENDPOINT, + Keys.MSI_TENANT, + Keys.MSI_ENDPOINT, + Keys.MSI_AUTHORITY, + Keys.USER_NAME, + Keys.USER_PASSWORD, + Keys.REFRESH_TOKEN, + Keys.REFRESH_TOKEN_ENDPOINT, + Keys.TOKEN_FILE, + Keys.CLIENT_ASSERTION_PROVIDER_TYPE, + Keys.SAS_FIXED_TOKEN) + + sealed trait Outcome + + /** Hadoop named a mechanism and every value it needs; `values` use the global key names. */ + final case class Resolved( + authType: String, + providerClass: Option[String], + values: Map[String, String]) + extends Outcome + + /** No auth-bearing key resolves for the account. */ + case object Unconfigured extends Outcome + + /** + * Hadoop named a mechanism the native scan cannot run; `providerClass` was never instantiated. + * `assertionProviderClass` is the trimmed value of + * `fs.azure.account.oauth2.client.assertion.provider.type` that a Workload Identity + * configuration names on hadoop-azure 3.5.0 and later. + */ + final case class Declined( + authType: String, + providerClass: Option[String], + assertionProviderClass: Option[String] = None) + extends Outcome + + /** Hadoop (or the reflection onto it) threw; `message` is already redacted. */ + final case class Failed(exceptionClass: String, message: String) extends Outcome + + def resolve(hadoopConf: Configuration, uri: URI): Outcome = + resolve(hadoopConf, uri, AbfsReflection.handles(AbfsReflection.loaderFor(hadoopConf))) + + private[objectstore] def resolve( + hadoopConf: Configuration, + uri: URI, + handles: Try[Handles]): Outcome = { + val secrets = ArrayBuffer[String]() + // Set once the AbfsConfiguration exists, so a failure can read the credentials it skipped. + var openSession: Option[Session] = None + try { + handles match { + case Failure(e) => failed(uri, e, secrets) + case Success(h) => + authorityParts(uri) match { + case Left(failure) => failure + case Right((container, account)) => + val session = new Session( + h, + h.newAbfsConfiguration(hadoopConf, account, container, uri), + hadoopConf, + account, + secrets) + openSession = Some(session) + if (!session.isConfigured) { + Unconfigured + } else { + val authType: Enum[_] = h.getAuthType(session.abfsConf, account) + authType.name match { + case AuthTypes.SHARED_KEY => sharedKey(session, uri, account) + case AuthTypes.SAS => sas(session, authType) + case AuthTypes.OAUTH => oauth(session, authType) + // Custom and UserboundSASWithOAuth run Java providers per request; never + // instantiated here. + case other => Declined(other, None) + } + } + } + } + } catch { + // NonFatal alone misses NoSuchMethodError and friends, the reflection failures this must + // report as an error marker rather than let escape into planning. + case e @ (_: LinkageError | NonFatal(_)) => + openSession.foreach(_.recordSkippedSecrets()) + failed(uri, e, secrets) + } + } + + /** + * `(container, account)` exactly as `AzureBlobFileSystemStore.authorityParts`: the raw + * authority split once at the first `@`, so port, case and percent-encoding are kept. Failures + * carry Hadoop's exception class and message. + */ + private[objectstore] def authorityParts(uri: URI): Either[Failed, (String, String)] = { + Option(uri.getRawAuthority) match { + case None => + Left(Failed(Exceptions.INVALID_URI_AUTHORITY, s"$uri has invalid authority.")) + case Some(authority) if !authority.contains("@") => + Left(Failed(Exceptions.INVALID_URI_AUTHORITY, s"$uri has invalid authority.")) + case Some(authority) => + val parts = authority.split("@", 2) + if (parts.length < 2 || parts(0).isEmpty) { + Left( + Failed( + Exceptions.INVALID_URI, + s"Invalid URI '$uri' has a malformed authority, expected container name. " + + "Authority takes the form abfs://[@]")) + } else { + Right((parts(0), parts(1))) + } + } + } + + // One resolution's view of an AbfsConfiguration; records every credential it reads so a + // failure message can be redacted against them. Each password read may load a credential + // provider's keystore, so a successful resolution reads only what the mechanism needs. + private final class Session( + val h: Handles, + val abfsConf: AnyRef, + hadoopConf: Configuration, + account: String, + secrets: ArrayBuffer[String]) { + + def password(key: String): Option[String] = { + val value = h.getPasswordString(abfsConf, key) + value.foreach(v => recordSecret(v)) + value + } + + // Both spellings, since Hadoop quotes the raw value and the providers use the trimmed one. + private def recordSecret(value: String): Unit = + Seq(value, value.trim).filter(_.length >= MIN_REDACTED_LENGTH).foreach(secrets += _) + + /** `getTrimmedPasswordString`: blank means the default, then trimmed. */ + def trimmedPassword(key: String, default: String): String = + password(key).filterNot(StringUtils.isBlank).getOrElse(default).trim + + def storageAccountKey(): String = { + val key = h.getStorageAccountKey(abfsConf) + recordSecret(key) + key + } + + // Stops at the first key that is set, plain keys first since they need no password lookup. + def isConfigured: Boolean = + plainAuthKeys.exists(isSet(abfsConf, _, password = false)) || + passwordAuthKeys.exists(isSet(abfsConf, _, password = true)) || + hasKeyProviderAccountKey + + /** + * Reads every credential key the resolution may have skipped, so the failure message is + * redacted against all of them. A read that throws is ignored: the failure being reported is + * the one that matters. + */ + def recordSkippedSecrets(): Unit = { + passwordAuthKeys.foreach(key => bestEffort(isSet(abfsConf, key, password = true))) + bestEffort(hasKeyProviderAccountKey) + } + + private def bestEffort(read: => Boolean): Unit = + try { + read + () + } catch { + case _: LinkageError | NonFatal(_) => () + } + + // A value Hadoop cannot read still counts as set: Hadoop reads a key only when the mechanism + // it picks needs it, and then fails in its own words. + private def isSet(conf: AnyRef, key: String, password: Boolean): Boolean = + h.read(conf, key, password) match { + case Right(value) => + if (password) value.foreach(v => recordSecret(v)) + value.isDefined + case Left(_) => true + } + + // SimpleKeyProvider reads the key through its own two-argument AbfsConfiguration, whose + // container level on 3.4.2+ is the literal `key.null.`; a key placed there + // configures the account for Hadoop, so it counts here too. + private def hasKeyProviderAccountKey: Boolean = + h.hasContainerScopedConfiguration && isSet( + h.newKeyProviderConfiguration(hadoopConf, account), + Keys.ACCOUNT_KEY, + password = true) + } + + private def sharedKey(session: Session, uri: URI, account: String): Outcome = { + // Store.initializeClient checks the account name before fetching the key. + if (account.indexOf('.') <= 0) { + Failed(Exceptions.INVALID_URI, s"Invalid URI $uri - account name is not fully qualified.") + } else { + val key = session.storageAccountKey() + if (key.isEmpty) { + // SharedKeyCredentials rejects the empty key SimpleKeyProvider's Base64 check let through. + Failed(classOf[IllegalArgumentException].getName, "Invalid account key.") + } else { + Resolved(AuthTypes.SHARED_KEY, None, Map(Keys.ACCOUNT_KEY -> key)) + } + } + } + + private def sas(session: Session, authType: Enum[_]): Outcome = { + val h = session.h + h.getTokenProviderClass( + session.abfsConf, + authType, + Keys.SAS_TOKEN_PROVIDER_TYPE, + h.sasTokenProviderClass) match { + case Some(custom) => Declined(AuthTypes.SAS, Some(custom.getName)) + case None => + // Hadoop's own checks: 3.3.4 has no fixed token and fails here, 3.4.1+ requires one. + h.getSASTokenProvider(session.abfsConf) + val fixed = session.trimmedPassword(Keys.SAS_FIXED_TOKEN, "") + Resolved(AuthTypes.SAS, None, Map(Keys.SAS_FIXED_TOKEN -> fixed)) + } + } + + private def oauth(session: Session, authType: Enum[_]): Outcome = { + val h = session.h + val providerClass = h.getTokenProviderClass( + session.abfsConf, + authType, + Keys.OAUTH_PROVIDER_TYPE, + h.accessTokenProviderClass) + val className = providerClass.map(_.getName) + val assertionProvider = + if (h.hasClientAssertionProvider && className.contains(Providers.WORKLOAD_IDENTITY)) { + session.password(Keys.CLIENT_ASSERTION_PROVIDER_TYPE).map(_.trim).filter(_.nonEmpty) + } else { + None + } + assertionProvider match { + case Some(assertion) => + // 3.5.0 Workload Identity with a custom ClientAssertionProvider: user code per request. + Declined(AuthTypes.OAUTH, className, Some(assertion)) + case None => + // Hadoop validates the configuration and throws in its own words: an unresolved class, + // a class that is not one of its five built-ins (matched by `==`, never instantiated), + // a missing mandatory key. The built-in constructors do no I/O. + h.getTokenProvider(session.abfsConf) + Resolved(AuthTypes.OAUTH, className, oauthValues(session, className.getOrElse(""))) + } + } + + // The keys each built-in provider's constructor receives, in the form Hadoop hands them over: + // mandatory and plain keys verbatim, defaulted keys trimmed with the AuthConfigurations default. + private def oauthValues(session: Session, providerClass: String): Map[String, String] = { + val h = session.h + def plain(key: String): Map[String, String] = session.password(key).map(key -> _).toMap + def defaulted(key: String, defaultField: String): Map[String, String] = + Map(key -> session.trimmedPassword(key, h.authDefault(defaultField))) + def authority(): Map[String, String] = { + val value = session.trimmedPassword( + Keys.MSI_AUTHORITY, + h.authDefault("DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY")) + Map(Keys.MSI_AUTHORITY -> appendSlashIfNeeded(value)) + } + + providerClass match { + case Providers.CLIENT_CREDS => + plain(Keys.CLIENT_ENDPOINT) ++ plain(Keys.CLIENT_ID) ++ plain(Keys.CLIENT_SECRET) + case Providers.USER_PASSWORD => + plain(Keys.CLIENT_ENDPOINT) ++ plain(Keys.USER_NAME) ++ plain(Keys.USER_PASSWORD) + case Providers.MSI => + defaulted(Keys.MSI_ENDPOINT, "DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT") ++ + plain(Keys.MSI_TENANT) ++ plain(Keys.CLIENT_ID) ++ authority() + case Providers.REFRESH_TOKEN => + defaulted( + Keys.REFRESH_TOKEN_ENDPOINT, + "DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT") ++ + plain(Keys.REFRESH_TOKEN) ++ plain(Keys.CLIENT_ID) + case Providers.WORKLOAD_IDENTITY => + authority() ++ plain(Keys.MSI_TENANT) ++ plain(Keys.CLIENT_ID) ++ + defaulted(Keys.TOKEN_FILE, "DEFAULT_FS_AZURE_ACCOUNT_OAUTH_TOKEN_FILE") + case _ => Map.empty + } + } + + private def appendSlashIfNeeded(authority: String): String = + if (authority.endsWith("/")) authority else authority + "/" + + /** + * The options for the native store: markers, plus the resolved values for `Resolved`. The + * provider marker is `comet.azure.sas.provider.class` for SAS and + * `comet.azure.oauth.provider.class` otherwise; a Workload Identity decline over a custom + * client assertion keeps the provider marker and adds + * `comet.azure.oauth.assertion.provider.class`. + */ + def toOptions(outcome: Outcome): Map[String, String] = outcome match { + case Resolved(authType, providerClass, values) => + Map(Markers.RESOLUTION -> Markers.RESOLVED, Markers.AUTH_TYPE -> authType) ++ + providerClass.map(providerMarker(authType) -> _) ++ values + case Unconfigured => + Map(Markers.RESOLUTION -> Markers.NONE) + case Declined(authType, providerClass, assertionProviderClass) => + Map(Markers.RESOLUTION -> Markers.DECLINED, Markers.AUTH_TYPE -> authType) ++ + providerClass.map(providerMarker(authType) -> _) ++ + assertionProviderClass.map(Markers.OAUTH_ASSERTION_PROVIDER_CLASS -> _) + case Failed(exceptionClass, message) => + Map( + Markers.RESOLUTION -> Markers.ERROR, + Markers.ERROR_CLASS -> exceptionClass, + Markers.ERROR_MESSAGE -> message) + } + + private def providerMarker(authType: String): String = + if (authType == AuthTypes.SAS) Markers.SAS_PROVIDER_CLASS else Markers.OAUTH_PROVIDER_CLASS + + /** + * Hides every `secret` of at least [[MIN_REDACTED_LENGTH]] characters in `message`, and the raw + * key a `SimpleKeyProvider` message quotes. + */ + private[objectstore] def redact(message: String, secrets: Iterable[String]): String = { + val withoutQuotedKey = QUOTED_KEY_PATTERN.replaceAllIn(message, "key =\"" + REDACTED + "\"") + secrets + .filter(_.length >= MIN_REDACTED_LENGTH) + .foldLeft(withoutQuotedKey)((text, secret) => text.replace(secret, REDACTED)) + } + + private def failed(uri: URI, e: Throwable, secrets: Iterable[String]): Failed = { + val chain = causeChain(e) + val thrown = chain.head + def describe(t: Throwable, verbatim: Boolean): String = { + val name = t.getClass.getName + if (verbatim) s"$name: ${redact(Option(t.getMessage).getOrElse(""), secrets)}" else name + } + logWarning( + s"Hadoop ABFS authentication for ${uri.getScheme}://${uri.getRawAuthority} failed on the " + + s"driver: ${chain.map(describe(_, verbatim = true)).mkString(", caused by ")}") + + val forwarded = chain.map(t => describe(t, isVerbatim(t))) + val hidden = chain.exists(t => !isVerbatim(t)) + val pointer = if (hidden) "; the full message is in the Spark driver log" else "" + val message = thrown match { + case _: ClassNotFoundException => + "hadoop-azure is not on the driver classpath; the native ABFS scan resolves " + + s"authentication through it (${forwarded.mkString(", caused by ")})" + case _ => forwarded.mkString(", caused by ") + pointer + } + Failed(thrown.getClass.getName, message) + } + + // KeyProviderException drops its cause, so its own message is all there is at that level. + private def causeChain(e: Throwable): Seq[Throwable] = { + val chain = ArrayBuffer[Throwable](e) + var cause = e.getCause + while (cause != null && chain.length < MAX_CAUSE_DEPTH && !chain.contains(cause)) { + chain += cause + cause = cause.getCause + } + chain.toSeq + } +} diff --git a/spark/src/main/scala/org/apache/comet/objectstore/AbfsReflection.scala b/spark/src/main/scala/org/apache/comet/objectstore/AbfsReflection.scala new file mode 100644 index 00000000000..9d9c5d0e2d0 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/objectstore/AbfsReflection.scala @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.objectstore + +import java.lang.reflect.{Constructor, InvocationTargetException, Method} +import java.net.URI + +import scala.util.{Failure, Success, Try} +import scala.util.control.NonFatal + +import org.apache.hadoop.conf.Configuration + +import org.apache.comet.util.ClassLoaders + +/** + * Reflective access to the hadoop-azure on the classpath. Comet does not link against + * hadoop-azure; every version-dependent surface of `AbfsConfiguration` is resolved here, once per + * class loader, so the resolver in [[AbfsAuthResolver]] can ask Hadoop the same questions + * `AzureBlobFileSystemStore.initializeClient` asks. + */ +private[objectstore] object AbfsReflection { + + object ClassNames { + val ABFS_CONFIGURATION = "org.apache.hadoop.fs.azurebfs.AbfsConfiguration" + val ABFS_SERVICE_TYPE = "org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType" + val AUTH_CONFIGURATIONS = "org.apache.hadoop.fs.azurebfs.constants.AuthConfigurations" + val AUTH_TYPE = "org.apache.hadoop.fs.azurebfs.services.AuthType" + val ACCESS_TOKEN_PROVIDER = "org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider" + val SAS_TOKEN_PROVIDER = "org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider" + // Present from hadoop-azure 3.4.1: the fixed SAS token path. + val FIXED_SAS_TOKEN_PROVIDER = "org.apache.hadoop.fs.azurebfs.services.FixedSASTokenProvider" + // Present from hadoop-azure 3.5.0: Workload Identity with a custom client assertion. + val CLIENT_ASSERTION_PROVIDER = "org.apache.hadoop.fs.azurebfs.oauth2.ClientAssertionProvider" + } + + // Store.getAbfsServiceTypeFromUrl: BLOB when the URI text contains this, DFS otherwise. + private val BLOB_DOMAIN_MARKER = ".blob." + + /** + * The operations the resolver needs from an `AbfsConfiguration`. A `Configuration` object is + * passed as `AnyRef` because its class is only known at run time. Hadoop's own exceptions + * propagate unwrapped from every method. + */ + trait Handles { + + /** True when hadoop-azure takes the container name (the 3.4.2+ four-argument constructor). */ + def hasContainerScopedConfiguration: Boolean + + /** True when `fs.azure.sas.fixed.token` exists (hadoop-azure 3.4.1+). */ + def hasFixedSasTokenProvider: Boolean + + /** True when a Workload Identity client assertion provider can be configured (3.5.0+). */ + def hasClientAssertionProvider: Boolean + + def accessTokenProviderClass: Class[_] + + def sasTokenProviderClass: Class[_] + + /** + * `new AbfsConfiguration(conf, account, container, serviceType)`, or the two-argument form. + */ + def newAbfsConfiguration( + conf: Configuration, + account: String, + container: String, + uri: URI): AnyRef + + /** + * `new AbfsConfiguration(conf, account)`: the form `SimpleKeyProvider` reads the key with. + */ + def newKeyProviderConfiguration(conf: Configuration, account: String): AnyRef + + /** `getAuthType(account)`: the `AuthType` enum constant. */ + def getAuthType(abfsConf: AnyRef, account: String): Enum[_] + + /** + * `getPasswordString(key)`: container (3.4.2+), account, then global, via credential + * providers. + */ + def getPasswordString(abfsConf: AnyRef, key: String): Option[String] + + /** + * `getPasswordString(key)` when `password` is set, otherwise `get(key)` (account-specific, + * then global). A value Hadoop cannot read (a `${...}` cycle, a failing credential provider) + * is returned as `Left` rather than thrown; fatal errors and linkage or reflection failures + * still propagate. + */ + def read(abfsConf: AnyRef, key: String, password: Boolean): Either[Throwable, Option[String]] + + /** + * `getTokenProviderClass(authType, key, null, xface)`: the provider class Hadoop would load. + */ + def getTokenProviderClass( + abfsConf: AnyRef, + authType: Enum[_], + key: String, + xface: Class[_]): Option[Class[_]] + + /** `getStorageAccountKey()`: runs the configured `KeyProvider`, as Hadoop does at FS init. */ + def getStorageAccountKey(abfsConf: AnyRef): String + + /** `getTokenProvider()`, called only for Hadoop's validation of a built-in OAuth provider. */ + def getTokenProvider(abfsConf: AnyRef): Unit + + /** `getSASTokenProvider()`, called only for Hadoop's validation of the fixed SAS token. */ + def getSASTokenProvider(abfsConf: AnyRef): Unit + + /** A `String` constant of `AuthConfigurations`, by field name. */ + def authDefault(fieldName: String): String + } + + // Keyed by the AbfsConfiguration class itself: a ClassValue holds nothing that pins a loader, + // and `computeValue` throwing caches nothing, so a loader that gains hadoop-azure later + // (ADD JAR) is probed again on the next call. + private val cache = new ClassValue[Handles] { + override protected def computeValue(abfsConfigurationClass: Class[_]): Handles = + new ReflectiveHandles(abfsConfigurationClass) + } + + /** The loader Hadoop's `FileSystem.get` would instantiate the ABFS classes with. */ + def loaderFor(hadoopConf: Configuration): ClassLoader = + Option(hadoopConf.getClassLoader) + .getOrElse(ClassLoaders.contextOrDefault(getClass.getClassLoader)) + + /** The handles for `loader`'s hadoop-azure, or the failure that keeps it from loading. */ + def handles(loader: ClassLoader): Try[Handles] = + attempt(cache.get(loadClass(loader, ClassNames.ABFS_CONFIGURATION))) + + // `Try` lets a `LinkageError` through, and a hadoop-azure whose own dependencies are missing + // fails with exactly that (`NoClassDefFoundError`), so it is caught here alongside `NonFatal`. + private def attempt[A](body: => A): Try[A] = { + try { + Success(body) + } catch { + case e @ (_: LinkageError | NonFatal(_)) => Failure(e) + } + } + + private def loadClass(loader: ClassLoader, name: String): Class[_] = + ClassLoaders.loadClass(name, loader) + + // Sibling classes come from the loader that defined AbfsConfiguration, the one hadoop-azure + // itself links against. + private final class ReflectiveHandles(abfsConfigurationClass: Class[_]) extends Handles { + private val loader = abfsConfigurationClass.getClassLoader + private val authTypeClass = loadClass(loader, ClassNames.AUTH_TYPE) + private val authConfigurationsClass = loadClass(loader, ClassNames.AUTH_CONFIGURATIONS) + + override val accessTokenProviderClass: Class[_] = + loadClass(loader, ClassNames.ACCESS_TOKEN_PROVIDER) + override val sasTokenProviderClass: Class[_] = + loadClass(loader, ClassNames.SAS_TOKEN_PROVIDER) + + private def isLoadable(name: String): Boolean = attempt(loadClass(loader, name)).isSuccess + + override val hasFixedSasTokenProvider: Boolean = isLoadable( + ClassNames.FIXED_SAS_TOKEN_PROVIDER) + override val hasClientAssertionProvider: Boolean = + isLoadable(ClassNames.CLIENT_ASSERTION_PROVIDER) + + // (Configuration, account, container, AbfsServiceType) exists from 3.4.2; the service type + // enum arrived with it, so a missing enum means the two-argument constructor. + private val serviceTypeClass: Option[Class[_]] = + attempt(loadClass(loader, ClassNames.ABFS_SERVICE_TYPE)).toOption + private val containerConstructor: Option[Constructor[_]] = serviceTypeClass.flatMap { st => + val ctor: Try[Constructor[_]] = attempt( + abfsConfigurationClass + .getConstructor(classOf[Configuration], classOf[String], classOf[String], st)) + ctor.toOption + } + private val accountConstructor: Constructor[_] = + abfsConfigurationClass.getConstructor(classOf[Configuration], classOf[String]) + + private val getAuthTypeMethod = + abfsConfigurationClass.getMethod("getAuthType", classOf[String]) + private val getMethod = abfsConfigurationClass.getMethod("get", classOf[String]) + private val getPasswordStringMethod = + abfsConfigurationClass.getMethod("getPasswordString", classOf[String]) + private val getTokenProviderClassMethod = abfsConfigurationClass.getMethod( + "getTokenProviderClass", + authTypeClass, + classOf[String], + classOf[Class[_]], + classOf[Class[_]]) + private val getStorageAccountKeyMethod = + abfsConfigurationClass.getMethod("getStorageAccountKey") + private val getTokenProviderMethod = abfsConfigurationClass.getMethod("getTokenProvider") + private val getSASTokenProviderMethod = + abfsConfigurationClass.getMethod("getSASTokenProvider") + + override def hasContainerScopedConfiguration: Boolean = containerConstructor.isDefined + + override def newAbfsConfiguration( + conf: Configuration, + account: String, + container: String, + uri: URI): AnyRef = { + (containerConstructor, serviceTypeClass) match { + case (Some(ctor), Some(st)) => + val serviceTypeName = if (uri.toString.contains(BLOB_DOMAIN_MARKER)) "BLOB" else "DFS" + construct(ctor, conf, account, container, enumConstant(st, serviceTypeName)) + case _ => + construct(accountConstructor, conf, account) + } + } + + override def newKeyProviderConfiguration(conf: Configuration, account: String): AnyRef = + construct(accountConstructor, conf, account) + + override def getAuthType(abfsConf: AnyRef, account: String): Enum[_] = + invoke(getAuthTypeMethod, abfsConf, account).asInstanceOf[Enum[_]] + + override def getPasswordString(abfsConf: AnyRef, key: String): Option[String] = + Option(invoke(getPasswordStringMethod, abfsConf, key)).map(_.asInstanceOf[String]) + + override def read( + abfsConf: AnyRef, + key: String, + password: Boolean): Either[Throwable, Option[String]] = { + val method = if (password) getPasswordStringMethod else getMethod + try { + Right(Option(method.invoke(abfsConf, key)).map(_.asInstanceOf[String])) + } catch { + // NonFatal excludes LinkageError, which rethrows below like any other failure. + case e: InvocationTargetException if e.getCause != null && NonFatal(e.getCause) => + Left(e.getCause) + case e: InvocationTargetException => throw unwrapped(e) + } + } + + override def getTokenProviderClass( + abfsConf: AnyRef, + authType: Enum[_], + key: String, + xface: Class[_]): Option[Class[_]] = { + Option(invoke(getTokenProviderClassMethod, abfsConf, authType, key, null, xface)) + .map(_.asInstanceOf[Class[_]]) + } + + override def getStorageAccountKey(abfsConf: AnyRef): String = + invoke(getStorageAccountKeyMethod, abfsConf).asInstanceOf[String] + + override def getTokenProvider(abfsConf: AnyRef): Unit = { + invoke(getTokenProviderMethod, abfsConf) + () + } + + override def getSASTokenProvider(abfsConf: AnyRef): Unit = { + invoke(getSASTokenProviderMethod, abfsConf) + () + } + + override def authDefault(fieldName: String): String = + authConfigurationsClass.getField(fieldName).get(null).asInstanceOf[String] + + private def enumConstant(enumClass: Class[_], name: String): AnyRef = + enumClass.getEnumConstants + .find(_.asInstanceOf[Enum[_]].name == name) + .getOrElse(throw new NoSuchFieldException(s"${enumClass.getName}.$name")) + .asInstanceOf[AnyRef] + + // Reflection wraps whatever the target throws; hand the resolver Hadoop's own exception. + private def invoke(method: Method, target: AnyRef, args: AnyRef*): AnyRef = { + try { + method.invoke(target, args: _*) + } catch { + case e: InvocationTargetException => throw unwrapped(e) + } + } + + private def construct(ctor: Constructor[_], args: AnyRef*): AnyRef = { + try { + ctor.newInstance(args: _*).asInstanceOf[AnyRef] + } catch { + case e: InvocationTargetException => throw unwrapped(e) + } + } + + private def unwrapped(e: InvocationTargetException): Throwable = + Option(e.getCause).getOrElse(e) + } +} diff --git a/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala b/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala index 75328649fea..b7c09da4694 100644 --- a/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala +++ b/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala @@ -37,12 +37,10 @@ object NativeConfig { "s3a" -> Seq("fs.s3a."), // Google Cloud Storage configurations "gs" -> Seq("fs.gs."), - // Azure Blob Storage configurations (can use both prefixes) + // Azure Blob Storage configurations (can use both prefixes). ABFS (abfs, abfss) is not + // forwarded by prefix: `AbfsAuthResolver` resolves its authentication for the one account. "wasb" -> Seq("fs.azure.", "fs.wasb."), - "wasbs" -> Seq("fs.azure.", "fs.wasb."), - // Azure Data Lake Storage Gen2 (ABFS) configurations. Hadoop ABFS authentication keys - "abfs" -> Seq("fs.azure.", "fs.abfs."), - "abfss" -> Seq("fs.azure.", "fs.abfss.", "fs.abfs.")) + "wasbs" -> Seq("fs.azure.", "fs.wasb.")) // Some alias filesystems report the literal authority "default" when the URI has none (e.g. // `scheme:///bucket/key`); the real bucket is then promoted from the URL path. Keys under this @@ -188,6 +186,10 @@ object NativeConfig { * Extract object store configs (S3, GCS, Azure, ...) from the Hadoop configuration for native * DataFusion. Captures global and per-bucket keys; native code prefers per-bucket. * + * For `abfs`/`abfss` the `fs.azure.*` namespace is not copied. [[AbfsAuthResolver]] asks the + * classpath's hadoop-azure which mechanism and values apply to the URI's account and forwards + * them as `comet.azure.*` markers plus global-named keys. + * * A scheme listed in `COMET_S3_COMPLIANT_SCHEMES_KEY` reuses the `fs.s3a.*` surface, and its * vendor `fs...*` keys are translated to `fs.s3a.*` (see * `translateVendorKeys`) AFTER the `fs.s3a.*` pass, so real vendor values override conflicting @@ -214,6 +216,13 @@ object NativeConfig { } val s3CompliantSchemes = parseSchemeSet(s3CompliantRaw) + // ABFS: Hadoop resolves the mechanism and values for this URI's account on the driver, and + // only those travel to native (never another account's keys, never the whole namespace). + if (scheme == "abfs" || scheme == "abfss") { + return options.toMap ++ AbfsAuthResolver.toOptions( + AbfsAuthResolver.resolve(hadoopConf, uri)) + } + // Prefixes for this scheme; a configured S3-compliant alias reuses the fs.s3a.* surface. val prefixes = objectStoreConfigPrefixes.get(scheme).orElse { if (s3CompliantSchemes.contains(scheme)) objectStoreConfigPrefixes.get("s3a") else None @@ -250,6 +259,41 @@ object NativeConfig { options.toMap } + /** + * The object store options for a scan over `uris`, or the reason the native scan cannot read + * them. The options are those of the first URI. Native reads every file of a partition through + * the store built for the partition's first file, and an ABFS store is bound to one container + * of one account, so a scan whose ABFS files span more than one container or account, or that + * mixes ABFS and other schemes, is declined. + */ + def extractScanObjectStoreOptions( + hadoopConf: Configuration, + uris: Iterable[URI]): Either[String, Map[String, String]] = { + val iter = uris.iterator + if (!iter.hasNext) { + return Right(Map.empty) + } + val first = iter.next() + val firstIsAbfs = isAbfs(first) + while (iter.hasNext) { + val uri = iter.next() + if (isAbfs(uri) != firstIsAbfs) { + return Left( + "Native scan does not support a scan that mixes ABFS and other file systems " + + s"(${first.getScheme} and ${uri.getScheme})") + } + if (firstIsAbfs && uri.getRawAuthority != first.getRawAuthority) { + return Left( + "Native scan does not support ABFS files from more than one container or account " + + s"(${first.getRawAuthority} and ${uri.getRawAuthority})") + } + } + Right(extractObjectStoreOptions(hadoopConf, first)) + } + + private def isAbfs(uri: URI): Boolean = + lowerScheme(uri).exists(scheme => scheme == "abfs" || scheme == "abfss") + /** * The value Hadoop's own consumers observe for `key`. `Configuration#get` expands any `${...}` * variable reference in the stored value against other conf entries and system properties, diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index a47f1c2a6b4..569ccc277f7 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -443,7 +443,7 @@ case class CometExecRule(session: SparkSession, queryStagePrep: Boolean = false) convertToComet(scan, CometIcebergNativeScan).getOrElse(scan) case scan: CometBatchScanExec if scan.wrapped.scan.isInstanceOf[CSVScan] => - convertToComet(scan, CometCsvNativeScanExec).getOrElse(scan) + convertToComet(scan, CometCsvNativeScanExec).getOrElse(scan.wrapped) // Comet JVM + native scan for V1 and V2 case op if isCometScan(op) => diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index c7c9a31909d..176c4c595ae 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -585,7 +585,14 @@ case class CometScanRule(session: SparkSession) val effectiveUri = new java.net.URI(effectiveLocation) - val hadoopS3Options = NativeConfig.extractObjectStoreOptions(hadoopConf, effectiveUri) + // Only a scheme the native Iceberg reader can open is worth resolving credentials + // for: the ABFS resolver may run a KeyProvider, and the scan is rejected below anyway. + val hadoopS3Options = + if (CometScanRule.isIcebergReadableScheme(effectiveUri, s3CompliantSchemes)) { + NativeConfig.extractObjectStoreOptions(hadoopConf, effectiveUri) + } else { + Map.empty[String, String] + } // Promote the DATA bucket's per-bucket `fs.s3a.bucket..*` settings to global (what // the native FileIO reads), NOT the metadata bucket's: the FileIO opens data/delete diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala index 286937635b7..38e7cdbf2e4 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala @@ -251,6 +251,23 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with CometTypeS } if (scanTypes.length == scan.output.length) { + // Collect S3/cloud storage configurations + val hadoopConf = scan.relation.sparkSession.sessionState + .newHadoopConfWithOptions(scan.relation.options) + + // Use selectedPartitions (static) instead of getFilePartitions() because at planning time + // DPP subqueries haven't been resolved yet. Object store options don't depend on DPP. + val fileUris = scan.selectedPartitions.view.flatMap(_.files).map(_.getPath.toUri) + val objectStoreOptions = + NativeConfig.extractScanObjectStoreOptions(hadoopConf, fileUris) match { + case Right(options) => options + case Left(reason) => + // CometExecRule falls back to the wrapped Spark scan, so it carries the reason too. + withFallbackReason(scan, reason) + withFallbackReason(scan.wrapped, reason) + return None + } + commonBuilder.addAllFields(scanTypes.asJava) // Sink operators don't have children @@ -279,14 +296,6 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with CometTypeS return None } - // Extract object store options from first file (S3 configs apply to all files in scan). - // Use selectedPartitions (static) instead of getFilePartitions() because at planning time - // DPP subqueries haven't been resolved yet. Object store options don't depend on DPP. - val firstFileUri = scan.selectedPartitions - .flatMap(_.files.headOption) - .headOption - .map(_.getPath.toUri) - // Constant metadata columns (file_path, file_name, file_size, file_block_start, // file_block_length, file_modification_time) are known before opening the file and // constant for every row read from it, exactly like partition columns. Spark places @@ -356,18 +365,10 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with CometTypeS commonBuilder.setAllowTypePromotion(CometConf.COMET_SCHEMA_EVOLUTION_ENABLED) commonBuilder.setAllowTimestampLtzToNtz(CometConf.COMET_ALLOW_TIMESTAMP_LTZ_AS_NTZ) - // Collect S3/cloud storage configurations - val hadoopConf = scan.relation.sparkSession.sessionState - .newHadoopConfWithOptions(scan.relation.options) - commonBuilder.setEncryptionEnabled(CometParquetUtils.encryptionEnabled(hadoopConf)) - firstFileUri.foreach { uri => - val objectStoreOptions = - NativeConfig.extractObjectStoreOptions(hadoopConf, uri) - objectStoreOptions.foreach { case (key, value) => - commonBuilder.putObjectStoreOptions(key, value) - } + objectStoreOptions.foreach { case (key, value) => + commonBuilder.putObjectStoreOptions(key, value) } // Set common data in NativeScan (file_partition will be populated at execution time) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometCsvNativeScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometCsvNativeScanExec.scala index dc92648b8d5..3d9f3f773fc 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometCsvNativeScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometCsvNativeScanExec.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.execution.datasources.v2.csv.CSVScan import com.google.common.base.Objects import org.apache.comet.{CometConf, ConfigEntry} +import org.apache.comet.CometSparkSessionExtensions.withFallbackReason import org.apache.comet.objectstore.NativeConfig import org.apache.comet.serde.{CometOperatorSerde, OperatorOuterClass} import org.apache.comet.serde.OperatorOuterClass.Operator @@ -93,6 +94,18 @@ object CometCsvNativeScanExec extends CometOperatorSerde[CometBatchScanExec] { new CSVOptions(csvScan.options.asScala.toMap, columnPruning, timeZone) } val filePartitions = op.inputPartitions.map(_.asInstanceOf[FilePartition]) + val hadoopConf = + sessionState.newHadoopConfWithOptions(op.session.sparkContext.conf.getAll.toMap) + val fileUris = filePartitions.view.flatMap(_.files).map(_.pathUri) + val objectStoreOptions = + NativeConfig.extractScanObjectStoreOptions(hadoopConf, fileUris) match { + case Right(options) => options + case Left(reason) => + // CometExecRule falls back to the wrapped Spark scan, so it carries the reason too. + withFallbackReason(op, reason) + withFallbackReason(op.wrapped, reason) + return None + } val csvOptionsProto = csvOptions2Proto(options) val dataSchemaProto = schema2Proto(csvScan.dataSchema) val readSchemaFieldNames = csvScan.readDataSchema.fieldNames @@ -104,15 +117,6 @@ object CometCsvNativeScanExec extends CometOperatorSerde[CometBatchScanExec] { val partitionSchemaProto = schema2Proto(csvScan.readPartitionSchema) val partitionsProto = filePartitions.map(partition2Proto(_, csvScan.readPartitionSchema)) - val objectStoreOptions = filePartitions.headOption - .flatMap { partitionFile => - val hadoopConf = sessionState - .newHadoopConfWithOptions(op.session.sparkContext.conf.getAll.toMap) - partitionFile.files.headOption - .map(file => NativeConfig.extractObjectStoreOptions(hadoopConf, file.pathUri)) - } - .getOrElse(Map.empty) - csvScanBuilder.putAllObjectStoreOptions(objectStoreOptions.asJava) csvScanBuilder.setCsvOptions(csvOptionsProto) csvScanBuilder.addAllFilePartitions(partitionsProto.asJava) diff --git a/spark/src/test/java/org/apache/comet/hadoop/fs/FakeAbfsFileSystem.java b/spark/src/test/java/org/apache/comet/hadoop/fs/FakeAbfsFileSystem.java new file mode 100644 index 00000000000..fe032e0d9db --- /dev/null +++ b/spark/src/test/java/org/apache/comet/hadoop/fs/FakeAbfsFileSystem.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.hadoop.fs; + +import java.io.IOException; +import java.net.URI; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.RawLocalFileSystem; + +/** + * A local-disk-backed FileSystem for the {@code abfss} scheme: {@code abfss://c@a/} is the + * local file at {@code }, so a test can plan a scan over several ABFS containers and + * accounts, and run its Spark fallback, without Azure. + */ +public class FakeAbfsFileSystem extends RawLocalFileSystem { + + private URI uri; + + public FakeAbfsFileSystem() { + // Avoid `URI scheme is not "file"` error on + // RawLocalFileSystem$DeprecatedRawLocalFileStatus.getOwner + RawLocalFileSystem.useStatIfAvailable(); + } + + @Override + public void initialize(URI name, Configuration conf) throws IOException { + super.initialize(name, conf); + uri = URI.create(name.getScheme() + "://" + name.getRawAuthority()); + } + + @Override + public String getScheme() { + return "abfss"; + } + + @Override + public URI getUri() { + // The superclass constructor qualifies its working directory before initialize runs. + return uri != null ? uri : URI.create("abfss://uninitialized@fake.dfs.core.windows.net"); + } +} diff --git a/spark/src/test/resources/abfs-auth-parity/expected-3.3.4.json b/spark/src/test/resources/abfs-auth-parity/expected-3.3.4.json new file mode 100644 index 00000000000..6cc13e20926 --- /dev/null +++ b/spark/src/test/resources/abfs-auth-parity/expected-3.3.4.json @@ -0,0 +1,876 @@ +{ + "cases": { + "at-account-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-oauth-global-sharedkey": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-account-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-account-userbound": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-global-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-global-userbound": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-unset-no-keys": { + "outcome": "unconfigured", + "values": {} + }, + "at-unset-other-account-only": { + "outcome": "unconfigured", + "values": {} + }, + "cal-base64-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-base64-non-ascii": { + "errorClass": "java.lang.ArrayIndexOutOfBoundsException", + "outcome": "failed", + "values": {} + }, + "cal-container-account-key-only": { + "outcome": "unconfigured", + "values": {} + }, + "cal-container-oauth-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-empty-account-key": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-empty-mandatory-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "cal-endpoint-with-path-prefix": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.example.net/prefix/tenant-abc/oauth2/v2.0/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cal-global-enum-parsed-first": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-msi-no-tenant": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-msi-no-tenant-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-null-fsname-key": { + "outcome": "unconfigured", + "values": {} + }, + "cal-null-fsname-key-beats-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cal-oauth-keys-without-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-port-key-matches": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "cal-port-key-missing": { + "outcome": "unconfigured", + "values": {} + }, + "cal-sas-container-fixed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-sas-guard-with-fixed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-sas-guard-without-fixed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-sas-untrimmed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-userbound-sas": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-wi-client-assertion-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "cal-wi-on-334": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "cp-jceks-account-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-beats-clear-text": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-client-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cp-jceks-no-fallback": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-no-fallback-clear-text-only": { + "outcome": "unconfigured", + "values": {} + }, + "cp-no-fallback-with-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-account-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-account-type-global-provider": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-global-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-global-type-global-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-sas-account-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-account-type-global-provider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "guard-sas-global-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-global-type-global-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "oauth-clientcreds": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-container-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-global": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-mixed-scopes": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-secret": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-other-account-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-padded-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": " https://login.microsoftonline.com/tenant-abc/oauth2/token ", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-custom-no-provider-type": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-custom-not-adaptee": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-msi": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-blank-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-container-tenant": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-custom-endpoint-authority": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.example.net/", + "fs.azure.account.oauth2.msi.endpoint": "http://msi.example.net/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-no-provider-type": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-empty": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-built-in": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-refreshtoken": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.microsoftonline.com/Common/oauth2/token" + } + }, + "oauth-refreshtoken-custom-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.example.net/token" + } + }, + "oauth-refreshtoken-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-refreshtoken-no-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.user.name": "user@example.com", + "fs.azure.account.oauth2.user.password": "sha256:c55bf0309fee4de8125a00ee97699a19a269caa35afcc8d1058bbd6283167fc8" + } + }, + "oauth-userpassword-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-password": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-user": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-blank-assertion-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-container-token-file": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-no-client-id": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-no-tenant": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-padded-token-file": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-blank-fixed": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "sas-custom-and-fixed": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-custom-only": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-fixed-global": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "sas-fixed-only": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "sas-neither": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "sas-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-question-mark": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-beats-global": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-account-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-container-ignored-with-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-global-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-global-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-custom": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:5be524205bd6aee5594ca7d1288c421a4cca089d63453278d1f75277c6d382bc" + } + }, + "sk-keyprovider-custom-null": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-empty": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-not-a-keyprovider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-simple": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-unknown-class": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-echo": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-shell-script-missing": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-script-unset": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "subst-cross-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:41800564f8313f5a8d3f468f205a7c39b55f63fa6efdb2dddd0005441f3c44bb" + } + }, + "subst-env-default": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "fallback-client", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "subst-env-home": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-account-without-dot": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-blob-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-blob-in-path-only": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-dfs-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-empty-container": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-no-at": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-null-authority": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-percent-encoded-container": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-port-in-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "uri-second-at": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account-lower-case-key": { + "outcome": "unconfigured", + "values": {} + } + }, + "hadoopAzureVersion": "3.3.4" +} diff --git a/spark/src/test/resources/abfs-auth-parity/expected-3.4.1.json b/spark/src/test/resources/abfs-auth-parity/expected-3.4.1.json new file mode 100644 index 00000000000..98ef579fe81 --- /dev/null +++ b/spark/src/test/resources/abfs-auth-parity/expected-3.4.1.json @@ -0,0 +1,936 @@ +{ + "cases": { + "at-account-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-oauth-global-sharedkey": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-account-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-account-userbound": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-global-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-global-userbound": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-unset-no-keys": { + "outcome": "unconfigured", + "values": {} + }, + "at-unset-other-account-only": { + "outcome": "unconfigured", + "values": {} + }, + "cal-base64-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-base64-non-ascii": { + "errorClass": "java.lang.ArrayIndexOutOfBoundsException", + "outcome": "failed", + "values": {} + }, + "cal-container-account-key-only": { + "outcome": "unconfigured", + "values": {} + }, + "cal-container-oauth-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "cal-empty-account-key": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-empty-mandatory-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "cal-endpoint-with-path-prefix": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.example.net/prefix/tenant-abc/oauth2/v2.0/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cal-global-enum-parsed-first": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-msi-no-tenant": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token" + } + }, + "cal-msi-no-tenant-no-client-id": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token" + } + }, + "cal-null-fsname-key": { + "outcome": "unconfigured", + "values": {} + }, + "cal-null-fsname-key-beats-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cal-oauth-keys-without-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-port-key-matches": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "cal-port-key-missing": { + "outcome": "unconfigured", + "values": {} + }, + "cal-sas-container-fixed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "cal-sas-guard-with-fixed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-sas-guard-without-fixed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "cal-sas-untrimmed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-userbound-sas": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-wi-client-assertion-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "cal-wi-on-334": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "cp-jceks-account-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-beats-clear-text": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-client-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cp-jceks-no-fallback": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-no-fallback-clear-text-only": { + "outcome": "unconfigured", + "values": {} + }, + "cp-no-fallback-with-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-account-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-account-type-global-provider": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-global-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-global-type-global-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-sas-account-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-account-type-global-provider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "guard-sas-global-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-global-type-global-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "oauth-clientcreds": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-container-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-global": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-mixed-scopes": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-secret": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-other-account-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-padded-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": " https://login.microsoftonline.com/tenant-abc/oauth2/token ", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-custom-no-provider-type": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-custom-not-adaptee": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-msi": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-blank-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-container-tenant": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-custom-endpoint-authority": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.example.net/", + "fs.azure.account.oauth2.msi.endpoint": "http://msi.example.net/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-no-client-id": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-no-provider-type": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-empty": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-built-in": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-refreshtoken": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.microsoftonline.com/Common/oauth2/token" + } + }, + "oauth-refreshtoken-custom-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.example.net/token" + } + }, + "oauth-refreshtoken-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-refreshtoken-no-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.user.name": "user@example.com", + "fs.azure.account.oauth2.user.password": "sha256:c55bf0309fee4de8125a00ee97699a19a269caa35afcc8d1058bbd6283167fc8" + } + }, + "oauth-userpassword-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-password": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-user": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "oauth-wi-blank-assertion-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "oauth-wi-container-token-file": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "oauth-wi-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-no-tenant": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-padded-token-file": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/mnt/tokens/token" + } + }, + "sas-blank-fixed": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "sas-custom-and-fixed": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-custom-only": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-fixed-global": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "sas-fixed-only": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "sas-neither": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "sas-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-question-mark": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:4f629ccc08252cde45726db17f66f06819423bba3254103f893dce6a1e626fc8" + } + }, + "sk-account-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-beats-global": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-account-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-container-ignored-with-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-global-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-global-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-custom": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:5be524205bd6aee5594ca7d1288c421a4cca089d63453278d1f75277c6d382bc" + } + }, + "sk-keyprovider-custom-null": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-empty": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-not-a-keyprovider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-simple": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-unknown-class": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-echo": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-shell-script-missing": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-script-unset": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "subst-cross-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:41800564f8313f5a8d3f468f205a7c39b55f63fa6efdb2dddd0005441f3c44bb" + } + }, + "subst-env-default": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "fallback-client", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "subst-env-home": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-account-without-dot": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-blob-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-blob-in-path-only": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-dfs-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-empty-container": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-no-at": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-null-authority": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-percent-encoded-container": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-port-in-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "uri-second-at": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account-lower-case-key": { + "outcome": "unconfigured", + "values": {} + } + }, + "hadoopAzureVersion": "3.4.1" +} diff --git a/spark/src/test/resources/abfs-auth-parity/expected-3.4.2.json b/spark/src/test/resources/abfs-auth-parity/expected-3.4.2.json new file mode 100644 index 00000000000..d2a368a7998 --- /dev/null +++ b/spark/src/test/resources/abfs-auth-parity/expected-3.4.2.json @@ -0,0 +1,947 @@ +{ + "cases": { + "at-account-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-oauth-global-sharedkey": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-account-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-account-userbound": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-global-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-global-userbound": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-unset-no-keys": { + "outcome": "unconfigured", + "values": {} + }, + "at-unset-other-account-only": { + "outcome": "unconfigured", + "values": {} + }, + "cal-base64-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-base64-non-ascii": { + "errorClass": "java.lang.ArrayIndexOutOfBoundsException", + "outcome": "failed", + "values": {} + }, + "cal-container-account-key-only": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-container-oauth-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cal-empty-account-key": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-empty-mandatory-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "cal-endpoint-with-path-prefix": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.example.net/prefix/tenant-abc/oauth2/v2.0/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cal-global-enum-parsed-first": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-msi-no-tenant": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token" + } + }, + "cal-msi-no-tenant-no-client-id": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token" + } + }, + "cal-null-fsname-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:bb99e5bb3a756453730056b900774c14cb31358abd79a0903bf0e0349d0ade7a" + } + }, + "cal-null-fsname-key-beats-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:bb99e5bb3a756453730056b900774c14cb31358abd79a0903bf0e0349d0ade7a" + } + }, + "cal-oauth-keys-without-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-port-key-matches": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "cal-port-key-missing": { + "outcome": "unconfigured", + "values": {} + }, + "cal-sas-container-fixed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-sas-guard-with-fixed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-sas-guard-without-fixed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "cal-sas-untrimmed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-userbound-sas": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-wi-client-assertion-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "cal-wi-on-334": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "cp-jceks-account-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-beats-clear-text": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-client-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cp-jceks-no-fallback": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-no-fallback-clear-text-only": { + "outcome": "unconfigured", + "values": {} + }, + "cp-no-fallback-with-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-account-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-account-type-global-provider": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-global-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-global-type-global-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-sas-account-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-account-type-global-provider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "guard-sas-global-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-global-type-global-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "oauth-clientcreds": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-container-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:1c008528aad647fbd3246a750e3f57dd6877398fbf25416d9269571ee4f35129" + } + }, + "oauth-clientcreds-global": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-mixed-scopes": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-secret": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-other-account-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-padded-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": " https://login.microsoftonline.com/tenant-abc/oauth2/token ", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-custom-no-provider-type": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-custom-not-adaptee": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-msi": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-blank-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-container-tenant": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "container-tenant" + } + }, + "oauth-msi-custom-endpoint-authority": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.example.net/", + "fs.azure.account.oauth2.msi.endpoint": "http://msi.example.net/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-no-client-id": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-no-provider-type": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-empty": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-built-in": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-refreshtoken": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.microsoftonline.com/Common/oauth2/token" + } + }, + "oauth-refreshtoken-custom-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.example.net/token" + } + }, + "oauth-refreshtoken-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-refreshtoken-no-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.user.name": "user@example.com", + "fs.azure.account.oauth2.user.password": "sha256:c55bf0309fee4de8125a00ee97699a19a269caa35afcc8d1058bbd6283167fc8" + } + }, + "oauth-userpassword-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-password": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-user": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "oauth-wi-blank-assertion-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "oauth-wi-container-token-file": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/mnt/tokens/container-token" + } + }, + "oauth-wi-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-no-tenant": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-padded-token-file": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/mnt/tokens/token" + } + }, + "sas-blank-fixed": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "sas-custom-and-fixed": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-custom-only": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-fixed-global": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "sas-fixed-only": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "sas-neither": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "sas-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-question-mark": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:4f629ccc08252cde45726db17f66f06819423bba3254103f893dce6a1e626fc8" + } + }, + "sk-account-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-beats-global": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-account-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-container-ignored-with-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-global-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-global-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-custom": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:5be524205bd6aee5594ca7d1288c421a4cca089d63453278d1f75277c6d382bc" + } + }, + "sk-keyprovider-custom-null": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-empty": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-not-a-keyprovider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-simple": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-unknown-class": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-echo": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-shell-script-missing": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-script-unset": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "subst-cross-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:41800564f8313f5a8d3f468f205a7c39b55f63fa6efdb2dddd0005441f3c44bb" + } + }, + "subst-env-default": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "fallback-client", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "subst-env-home": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-account-without-dot": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-blob-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-blob-in-path-only": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-dfs-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-empty-container": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-no-at": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-null-authority": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-percent-encoded-container": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "container-client", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-port-in-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "uri-second-at": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account-lower-case-key": { + "outcome": "unconfigured", + "values": {} + } + }, + "hadoopAzureVersion": "3.4.2" +} diff --git a/spark/src/test/resources/abfs-auth-parity/expected-3.5.0.json b/spark/src/test/resources/abfs-auth-parity/expected-3.5.0.json new file mode 100644 index 00000000000..7197ec2fb7a --- /dev/null +++ b/spark/src/test/resources/abfs-auth-parity/expected-3.5.0.json @@ -0,0 +1,943 @@ +{ + "cases": { + "at-account-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-account-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-oauth-global-sharedkey": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-account-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-account-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-account-userbound": { + "authType": "UserboundSASWithOAuth", + "outcome": "declined", + "values": {} + }, + "at-global-custom": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "at-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-lower-case": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "at-global-oauth": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "at-global-sas-custom": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "at-global-sharedkey": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "at-global-userbound": { + "authType": "UserboundSASWithOAuth", + "outcome": "declined", + "values": {} + }, + "at-unset-no-keys": { + "outcome": "unconfigured", + "values": {} + }, + "at-unset-other-account-only": { + "outcome": "unconfigured", + "values": {} + }, + "cal-base64-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-base64-non-ascii": { + "errorClass": "java.lang.ArrayIndexOutOfBoundsException", + "outcome": "failed", + "values": {} + }, + "cal-container-account-key-only": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-container-oauth-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cal-empty-account-key": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-empty-mandatory-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "cal-endpoint-with-path-prefix": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.example.net/prefix/tenant-abc/oauth2/v2.0/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cal-global-enum-parsed-first": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "cal-msi-no-tenant": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token" + } + }, + "cal-msi-no-tenant-no-client-id": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token" + } + }, + "cal-null-fsname-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:bb99e5bb3a756453730056b900774c14cb31358abd79a0903bf0e0349d0ade7a" + } + }, + "cal-null-fsname-key-beats-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:bb99e5bb3a756453730056b900774c14cb31358abd79a0903bf0e0349d0ade7a" + } + }, + "cal-oauth-keys-without-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "cal-port-key-matches": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "cal-port-key-missing": { + "outcome": "unconfigured", + "values": {} + }, + "cal-sas-container-fixed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-sas-guard-with-fixed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-sas-guard-without-fixed-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "cal-sas-untrimmed-token": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "cal-userbound-sas": { + "authType": "UserboundSASWithOAuth", + "outcome": "declined", + "values": {} + }, + "cal-wi-client-assertion-provider": { + "assertionProviderClass": "com.example.comet.NeverLoadedAssertionProvider", + "authType": "OAuth", + "outcome": "declined", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": {} + }, + "cal-wi-on-334": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "cp-jceks-account-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-beats-clear-text": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-jceks-client-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "cp-jceks-no-fallback": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "cp-no-fallback-clear-text-only": { + "outcome": "unconfigured", + "values": {} + }, + "cp-no-fallback-with-auth-type": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-account-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-account-type-global-provider": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "guard-oauth-global-type-account-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-oauth-global-type-global-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "guard-sas-account-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-account-type-global-provider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "guard-sas-global-type-account-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "guard-sas-global-type-global-provider": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "oauth-clientcreds": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-container-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:1c008528aad647fbd3246a750e3f57dd6877398fbf25416d9269571ee4f35129" + } + }, + "oauth-clientcreds-global": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-mixed-scopes": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-no-secret": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-clientcreds-other-account-secret": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-clientcreds-padded-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": " https://login.microsoftonline.com/tenant-abc/oauth2/token ", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-custom-no-provider-type": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-custom-not-adaptee": { + "authType": "Custom", + "outcome": "declined", + "values": {} + }, + "oauth-msi": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-blank-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-container-tenant": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "container-tenant" + } + }, + "oauth-msi-custom-endpoint-authority": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.example.net/", + "fs.azure.account.oauth2.msi.endpoint": "http://msi.example.net/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-msi-no-client-id": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider", + "values": { + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.endpoint": "http://169.254.169.254/metadata/identity/oauth2/token", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc" + } + }, + "oauth-no-provider-type": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-empty": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-not-built-in": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "oauth-provider-padded": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "oauth-refreshtoken": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.microsoftonline.com/Common/oauth2/token" + } + }, + "oauth-refreshtoken-custom-endpoint": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.refresh.token": "sha256:e65009f6e0ae9fc204adcb73a208f63c582b7839bde7fe836da36a3df6ae7a85", + "fs.azure.account.oauth2.refresh.token.endpoint": "https://login.example.net/token" + } + }, + "oauth-refreshtoken-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-refreshtoken-no-token": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.user.name": "user@example.com", + "fs.azure.account.oauth2.user.password": "sha256:c55bf0309fee4de8125a00ee97699a19a269caa35afcc8d1058bbd6283167fc8" + } + }, + "oauth-userpassword-no-endpoint": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-password": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-userpassword-no-user": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "oauth-wi-blank-assertion-provider": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/var/run/secrets/azure/tokens/azure-identity-token" + } + }, + "oauth-wi-container-token-file": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/mnt/tokens/container-token" + } + }, + "oauth-wi-no-client-id": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-no-tenant": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException", + "outcome": "failed", + "values": {} + }, + "oauth-wi-padded-token-file": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", + "values": { + "fs.azure.account.oauth2.client.id": "client-123", + "fs.azure.account.oauth2.msi.authority": "https://login.microsoftonline.com/", + "fs.azure.account.oauth2.msi.tenant": "tenant-abc", + "fs.azure.account.oauth2.token.file": "/mnt/tokens/token" + } + }, + "sas-blank-fixed": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "sas-custom-and-fixed": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-custom-only": { + "authType": "SAS", + "outcome": "declined", + "providerClass": "org.apache.comet.objectstore.ParitySasTokenProvider", + "values": {} + }, + "sas-fixed-global": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "sas-fixed-only": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:332a89b8fbd21ed8631f462437d07dcc560375e25159ff1db89cdb7945c0bd1b" + } + }, + "sas-neither": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException", + "outcome": "failed", + "values": {} + }, + "sas-provider-missing-class": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-provider-not-a-provider": { + "errorClass": "java.lang.RuntimeException", + "outcome": "failed", + "values": {} + }, + "sas-question-mark": { + "authType": "SAS", + "outcome": "resolved", + "values": { + "fs.azure.sas.fixed.token": "sha256:4f629ccc08252cde45726db17f66f06819423bba3254103f893dce6a1e626fc8" + } + }, + "sk-account-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-beats-global": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-account-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-account-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-account-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-container-ignored-with-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-global-bad-char": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-bad-length": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-empty": { + "errorClass": "java.lang.IllegalArgumentException", + "outcome": "failed", + "values": {} + }, + "sk-global-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-global-valid": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-custom": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:5be524205bd6aee5594ca7d1288c421a4cca089d63453278d1f75277c6d382bc" + } + }, + "sk-keyprovider-custom-null": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-empty": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-not-a-keyprovider": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-padded": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-keyprovider-simple": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-keyprovider-unknown-class": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-echo": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "sk-shell-script-missing": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "sk-shell-script-unset": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException", + "outcome": "failed", + "values": {} + }, + "subst-cross-key": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:41800564f8313f5a8d3f468f205a7c39b55f63fa6efdb2dddd0005441f3c44bb" + } + }, + "subst-env-default": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "fallback-client", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "subst-env-home": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-account-without-dot": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-blob-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-blob-in-path-only": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-dfs-host": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-empty-container": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException", + "outcome": "failed", + "values": {} + }, + "uri-no-at": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-null-authority": { + "errorClass": "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException", + "outcome": "failed", + "values": {} + }, + "uri-percent-encoded-container": { + "authType": "OAuth", + "outcome": "resolved", + "providerClass": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider", + "values": { + "fs.azure.account.oauth2.client.endpoint": "https://login.microsoftonline.com/tenant-abc/oauth2/token", + "fs.azure.account.oauth2.client.id": "container-client", + "fs.azure.account.oauth2.client.secret": "sha256:c1a39329c3e1799f02b7067ae57bbece8f4accbc5b952cc94fe97799332f1b27" + } + }, + "uri-port-in-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:e1e799c979ec65dcc29a55d9fe8d6dba055a00b83ababed9600d2ea0462f4fe3" + } + }, + "uri-second-at": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account": { + "authType": "SharedKey", + "outcome": "resolved", + "values": { + "fs.azure.account.key": "sha256:1c1185e02ff3e23b3e5a1c5bc86cf15d4126caa3dcde0fdb6e93adc4deec119e" + } + }, + "uri-upper-case-account-lower-case-key": { + "outcome": "unconfigured", + "values": {} + } + }, + "hadoopAzureVersion": "3.5.0" +} diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParityCases.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParityCases.scala new file mode 100644 index 00000000000..73652d0cee4 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParityCases.scala @@ -0,0 +1,1014 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.objectstore + +import java.net.URI +import java.util.{Date, Locale} + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.azurebfs.extensions.{CustomTokenProviderAdaptee, SASTokenProvider} +import org.apache.hadoop.fs.azurebfs.services.KeyProvider + +import org.apache.comet.objectstore.AbfsAuthResolver.{Keys, Providers} + +/** + * The configurations [[AbfsAuthParitySuite]] runs through the resolver and through the real + * `AzureBlobFileSystem`. Built in code so nothing checked in can drift from the cases; the + * recorded expectation per hadoop-azure version lives in `abfs-auth-parity/expected-*.json`. + * + * A case may carry `pins`: the outcome kind and error class that reading the hadoop-azure source + * predicts per version. Every `calibration` case has one for all four versions, so the recorded + * files are checked against that reading rather than trusted. + */ +object AbfsAuthParityCases { + + val VERSIONS: Seq[String] = Seq("3.3.4", "3.4.1", "3.4.2", "3.5.0") + + object Outcomes { + val RESOLVED = "resolved" + val UNCONFIGURED = "unconfigured" + val DECLINED = "declined" + val FAILED = "failed" + } + + object ExceptionClasses { + private val pkg = "org.apache.hadoop.fs.azurebfs.contracts.exceptions." + val KEY_PROVIDER: String = pkg + "KeyProviderException" + val INVALID_URI: String = pkg + "InvalidUriException" + val INVALID_URI_AUTHORITY: String = pkg + "InvalidUriAuthorityException" + val TOKEN_ACCESS_PROVIDER: String = pkg + "TokenAccessProviderException" + val SAS_TOKEN_PROVIDER: String = pkg + "SASTokenProviderException" + val CONFIGURATION_PROPERTY_NOT_FOUND: String = pkg + "ConfigurationPropertyNotFoundException" + val ILLEGAL_ARGUMENT: String = classOf[IllegalArgumentException].getName + val RUNTIME: String = classOf[RuntimeException].getName + val ARRAY_INDEX: String = classOf[ArrayIndexOutOfBoundsException].getName + } + + /** The outcome the hadoop-azure source predicts for one version. */ + final case class Pin(outcome: String, errorClass: Option[String] = None) + + object Pin { + val resolved: Pin = Pin(Outcomes.RESOLVED) + val unconfigured: Pin = Pin(Outcomes.UNCONFIGURED) + val declined: Pin = Pin(Outcomes.DECLINED) + def failed(exceptionClass: String): Pin = Pin(Outcomes.FAILED, Some(exceptionClass)) + } + + /** + * @param liveValues + * forwarded values that depend on the machine (an environment variable); the suite checks + * them directly and records a placeholder instead. + */ + final case class ParityCase( + id: String, + group: String, + uri: URI, + conf: Map[String, String], + note: String, + pins: Map[String, Pin] = Map.empty, + liveValues: Map[String, String] = Map.empty) + + def pins(v334: Pin, v341: Pin, v342: Pin, v350: Pin): Map[String, Pin] = + Map("3.3.4" -> v334, "3.4.1" -> v341, "3.4.2" -> v342, "3.5.0" -> v350) + + def allVersions(pin: Pin): Map[String, Pin] = pins(pin, pin, pin, pin) + + /** `before` on 3.3.4, `after` from 3.4.1 (fixed SAS token, Workload Identity, optional MSI). */ + def from341(before: Pin, after: Pin): Map[String, Pin] = pins(before, after, after, after) + + /** `before` up to 3.4.1, `after` from 3.4.2 (container-scoped keys). */ + def from342(before: Pin, after: Pin): Map[String, Pin] = pins(before, before, after, after) + + /** `before` up to 3.4.2, `after` on 3.5.0 (user-bound SAS, client assertion provider). */ + def from350(before: Pin, after: Pin): Map[String, Pin] = pins(before, before, before, after) + + val ACCOUNT = "myacct.dfs.core.windows.net" + val CONTAINER = "data" + val URI_DFS = new URI(s"abfss://$CONTAINER@$ACCOUNT/path/file.parquet") + + // Base64 of "secret", "global", "container", "null", "custom", "port", "env". + val KEY_ACCOUNT = "c2VjcmV0" + val KEY_GLOBAL = "Z2xvYmFs" + val KEY_CONTAINER = "Y29udGFpbmVy" + val KEY_NULL_LITERAL = "bnVsbA==" + val KEY_CUSTOM = "Y3VzdG9t" + val KEY_PORT = "cG9ydA==" + val KEY_ENV = "ZW52" + + val ENDPOINT = "https://login.microsoftonline.com/tenant-abc/oauth2/token" + val CLIENT_ID = "client-123" + val CLIENT_SECRET = "client-secret-value" + val TENANT = "tenant-abc" + val FIXED_SAS = "sv=2020-08-04&sig=fixed-signature" + val DEFAULT_MSI_ENDPOINT = "http://169.254.169.254/metadata/identity/oauth2/token" + val DEFAULT_AUTHORITY = "https://login.microsoftonline.com/" + val DEFAULT_TOKEN_FILE = "/var/run/secrets/azure/tokens/azure-identity-token" + + val SIMPLE_KEY_PROVIDER = "org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider" + val SHELL_KEY_PROVIDER = "org.apache.hadoop.fs.azurebfs.services.ShellDecryptionKeyProvider" + val MISSING_CLASS = "com.example.comet.MissingProvider" + val NOT_A_PROVIDER: String = classOf[String].getName + val CUSTOM_ADAPTEE: String = classOf[ParityCustomTokenAdaptee].getName + val SAS_PROVIDER: String = classOf[ParitySasTokenProvider].getName + val FOREIGN_ACCESS_TOKEN_PROVIDER: String = + classOf[NeverInstantiatedAccessTokenProvider].getName + + /** The aliases the suite stores in its JCEKS keystore, with no clear-text counterpart. */ + val credentialEntries: Map[String, String] = + Map(acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT, acct(Keys.CLIENT_SECRET) -> CLIENT_SECRET) + + /** + * The variable `${env.COMET_ABFS_PARITY_UNSET}` falls back to; the suite asserts it is unset. + */ + val UNSET_ENV_VAR = "COMET_ABFS_PARITY_UNSET" + + def acct(key: String, account: String = ACCOUNT): String = s"$key.$account" + + def cont(key: String, container: String = CONTAINER, account: String = ACCOUNT): String = + s"$key.$container.$account" + + private def uriFor(container: String, account: String): URI = + new URI(s"abfss://$container@$account/path/file.parquet") + + private val clientCreds: Map[String, String] = Map( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.CLIENT_CREDS, + acct(Keys.CLIENT_ENDPOINT) -> ENDPOINT, + acct(Keys.CLIENT_ID) -> CLIENT_ID, + acct(Keys.CLIENT_SECRET) -> CLIENT_SECRET) + + private val msi: Map[String, String] = Map( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.MSI, + acct(Keys.MSI_TENANT) -> TENANT, + acct(Keys.CLIENT_ID) -> CLIENT_ID) + + private val workloadIdentity: Map[String, String] = Map( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.WORKLOAD_IDENTITY, + acct(Keys.MSI_TENANT) -> TENANT, + acct(Keys.CLIENT_ID) -> CLIENT_ID) + + private val refreshToken: Map[String, String] = Map( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.REFRESH_TOKEN, + acct(Keys.REFRESH_TOKEN) -> "refresh-token-value", + acct(Keys.CLIENT_ID) -> CLIENT_ID) + + private val userPassword: Map[String, String] = Map( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.USER_PASSWORD, + acct(Keys.CLIENT_ENDPOINT) -> ENDPOINT, + acct(Keys.USER_NAME) -> "user@example.com", + acct(Keys.USER_PASSWORD) -> "user-password-value") + + private val sasCustom: Map[String, String] = + Map(acct(Keys.AUTH_TYPE) -> "SAS", acct(Keys.SAS_TOKEN_PROVIDER_TYPE) -> SAS_PROVIDER) + + // 3.3.4 has no fixed SAS token, so a configuration that names no provider class fails there in + // TokenAccessProviderException and in SASTokenProviderException from 3.4.1. + private val fixedSasOnly: Map[String, Pin] = + from341(Pin.failed(ExceptionClasses.TOKEN_ACCESS_PROVIDER), Pin.resolved) + private val noSasSource: Map[String, Pin] = from341( + Pin.failed(ExceptionClasses.TOKEN_ACCESS_PROVIDER), + Pin.failed(ExceptionClasses.SAS_TOKEN_PROVIDER)) + + // WorkloadIdentityTokenProvider is absent from 3.3.4; Configuration.getClass raises the + // RuntimeException the resolver reports (Hadoop itself would wrap it once more). + private val fromWorkloadIdentity: Map[String, Pin] = + from341(Pin.failed(ExceptionClasses.RUNTIME), Pin.resolved) + + def cases(credentialProviderPath: String): Seq[ParityCase] = + uriCases ++ authTypeCases ++ sharedKeyCases ++ oauthCases ++ guardCases ++ sasCases ++ + credentialProviderCases(credentialProviderPath) ++ substitutionCases ++ calibrationCases + + private def c( + id: String, + group: String, + note: String, + conf: Map[String, String], + uri: URI = URI_DFS, + pins: Map[String, Pin] = Map.empty, + liveValues: Map[String, String] = Map.empty): ParityCase = + ParityCase(id, group, uri, conf, note, pins, liveValues) + + // ---- URI shapes ---- + + private def uriCases: Seq[ParityCase] = { + val blobAccount = "myacct.blob.core.windows.net" + val portAccount = s"$ACCOUNT:8080" + val upperAccount = "MyAcct.DFS.core.windows.net" + val encodedContainer = "my%2Ddata" + val secondAt = s"user@$ACCOUNT" + Seq( + c("uri-dfs-host", "uri", "plain dfs host", Map(acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT)), + // hns.enabled=false: 3.4.2+ reject a Blob endpoint for an HNS account after authentication. + c( + "uri-blob-host", + "uri", + ".blob. host selects the BLOB service type on 3.4.2+", + Map( + acct(Keys.ACCOUNT_KEY, blobAccount) -> KEY_ACCOUNT, + "fs.azure.account.hns.enabled" -> "false"), + uri = uriFor(CONTAINER, blobAccount)), + c( + "uri-blob-in-path-only", + "uri", + ".blob. in the path also selects BLOB (Store checks the whole URI text)", + Map(acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT, "fs.azure.account.hns.enabled" -> "false"), + uri = new URI(s"abfss://$CONTAINER@$ACCOUNT/x.blob.y/file.parquet")), + c( + "uri-port-in-account", + "uri", + "the port is part of the account name Hadoop looks keys up under", + Map( + acct(Keys.ACCOUNT_KEY, portAccount) -> KEY_PORT, + acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + uri = uriFor(CONTAINER, portAccount)), + c( + "uri-upper-case-account", + "uri", + "account name case is kept when matching keys", + Map(acct(Keys.ACCOUNT_KEY, upperAccount) -> KEY_ACCOUNT), + uri = uriFor(CONTAINER, upperAccount)), + c( + "uri-upper-case-account-lower-case-key", + "uri", + "a lower-case key does not configure a mixed-case account", + Map(acct(Keys.ACCOUNT_KEY, upperAccount.toLowerCase(Locale.ROOT)) -> KEY_ACCOUNT), + uri = uriFor(CONTAINER, upperAccount), + pins = allVersions(Pin.unconfigured)), + c( + "uri-percent-encoded-container", + "uri", + "container-scoped keys match the raw (still encoded) container name", + clientCreds ++ Map(cont(Keys.CLIENT_ID, encodedContainer) -> "container-client"), + uri = uriFor(encodedContainer, ACCOUNT)), + c( + "uri-second-at", + "uri", + "everything after the first '@' is the account name", + Map(acct(Keys.ACCOUNT_KEY, secondAt) -> KEY_ACCOUNT), + uri = uriFor(CONTAINER, secondAt)), + c( + "uri-no-at", + "uri", + "no '@' in the authority", + Map(Keys.ACCOUNT_KEY -> KEY_GLOBAL), + uri = new URI(s"abfss://$ACCOUNT/path/file.parquet"), + pins = allVersions(Pin.failed(ExceptionClasses.INVALID_URI_AUTHORITY))), + c( + "uri-empty-container", + "uri", + "empty container name", + Map(Keys.ACCOUNT_KEY -> KEY_GLOBAL), + uri = new URI(s"abfss://@$ACCOUNT/path/file.parquet"), + pins = allVersions(Pin.failed(ExceptionClasses.INVALID_URI))), + c( + "uri-null-authority", + "uri", + "no authority at all", + Map(Keys.ACCOUNT_KEY -> KEY_GLOBAL), + uri = new URI("abfss:///path/file.parquet"), + pins = allVersions(Pin.failed(ExceptionClasses.INVALID_URI_AUTHORITY))), + c( + "uri-account-without-dot", + "uri", + "SharedKey needs a fully qualified account name", + Map(acct(Keys.ACCOUNT_KEY, "myacct") -> KEY_ACCOUNT), + uri = uriFor(CONTAINER, "myacct"), + pins = allVersions(Pin.failed(ExceptionClasses.INVALID_URI)))) + } + + // ---- auth type values and scopes ---- + + private def authTypeCases: Seq[ParityCase] = { + val oauthKeys = clientCreds - acct(Keys.AUTH_TYPE) + val sharedKey = Map(acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT) + val badEnum = Pin.failed(ExceptionClasses.ILLEGAL_ARGUMENT) + def scoped(scope: String, key: String): String = + if (scope == "global") key else acct(key) + Seq("global", "account").flatMap { scope => + val at = scoped(scope, Keys.AUTH_TYPE) + Seq( + c( + s"at-$scope-sharedkey", + "auth-type", + s"SharedKey ($scope)", + sharedKey + (at -> "SharedKey")), + c(s"at-$scope-oauth", "auth-type", s"OAuth ($scope)", oauthKeys + (at -> "OAuth")), + c( + s"at-$scope-sas-custom", + "auth-type", + s"SAS with a custom provider ($scope)", + Map(at -> "SAS", acct(Keys.SAS_TOKEN_PROVIDER_TYPE) -> SAS_PROVIDER), + pins = allVersions(Pin.declined)), + c( + s"at-$scope-custom", + "auth-type", + s"Custom ($scope)", + Map(at -> "Custom", acct(Keys.OAUTH_PROVIDER_TYPE) -> CUSTOM_ADAPTEE), + pins = allVersions(Pin.declined)), + c( + s"at-$scope-lower-case", + "auth-type", + s"'oauth' is not an enum constant ($scope)", + oauthKeys + (at -> "oauth"), + pins = allVersions(badEnum)), + c( + s"at-$scope-padded", + "auth-type", + s"' OAuth ' is trimmed by getEnum ($scope)", + oauthKeys + (at -> " OAuth "), + pins = allVersions(Pin.resolved)), + c( + s"at-$scope-empty", + "auth-type", + s"'' is not unset: Enum.valueOf fails ($scope)", + oauthKeys + (at -> ""), + pins = allVersions(badEnum)), + c( + s"at-$scope-userbound", + "auth-type", + s"UserboundSASWithOAuth exists from 3.5.0 ($scope)", + Map(at -> "UserboundSASWithOAuth", acct(Keys.SAS_TOKEN_PROVIDER_TYPE) -> SAS_PROVIDER), + pins = from350(badEnum, Pin.declined))) + } ++ Seq( + c( + "at-account-oauth-global-sharedkey", + "auth-type", + "the account value wins over a valid global one", + clientCreds + (Keys.AUTH_TYPE -> "SharedKey") + (Keys.ACCOUNT_KEY -> KEY_GLOBAL), + pins = allVersions(Pin.resolved)), + c( + "at-unset-no-keys", + "auth-type", + "nothing configured for the account", + Map("fs.azure.account.hns.enabled" -> "true"), + pins = allVersions(Pin.unconfigured)), + c( + "at-unset-other-account-only", + "auth-type", + "another account's key does not configure this one", + Map(acct(Keys.ACCOUNT_KEY, "other.dfs.core.windows.net") -> KEY_GLOBAL), + pins = allVersions(Pin.unconfigured))) + } + + // ---- SharedKey ---- + + private def sharedKeyCases: Seq[ParityCase] = { + val kpe = Pin.failed(ExceptionClasses.KEY_PROVIDER) + val emptyKey = Pin.failed(ExceptionClasses.ILLEGAL_ARGUMENT) + val badKeys = Seq( + ("bad-length", "c2VjcmV", "length % 4 != 0", kpe), + ("bad-char", "c2Vj!mV0", "a character outside the Base64 alphabet", kpe), + ("padded", " c2VjcmV0 ", "whitespace is not trimmed, so the length is wrong", kpe), + ("empty", "", "'' passes the Base64 check and fails in SharedKeyCredentials", emptyKey)) + val placements = Seq(("account", acct(Keys.ACCOUNT_KEY)), ("global", Keys.ACCOUNT_KEY)) + val validity = placements.flatMap { case (placement, key) => + c( + s"sk-$placement-valid", + "sharedkey", + s"valid key ($placement)", + Map(key -> KEY_ACCOUNT)) +: + badKeys.map { case (suffix, value, note, pin) => + c( + s"sk-$placement-$suffix", + "sharedkey", + s"$note ($placement)", + Map(key -> value), + pins = allVersions(pin)) + } + } + val providers = Seq( + c( + "sk-keyprovider-simple", + "sharedkey", + "explicit SimpleKeyProvider", + Map( + acct(Keys.KEY_PROVIDER) -> SIMPLE_KEY_PROVIDER, + acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT)), + c( + "sk-keyprovider-unknown-class", + "sharedkey", + "keyprovider names a class that does not exist", + Map(acct(Keys.KEY_PROVIDER) -> MISSING_CLASS, acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(kpe)), + c( + "sk-keyprovider-not-a-keyprovider", + "sharedkey", + "keyprovider names a class that is not a KeyProvider", + Map(acct(Keys.KEY_PROVIDER) -> NOT_A_PROVIDER, acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(kpe)), + c( + "sk-keyprovider-empty", + "sharedkey", + "keyprovider '' is a class name Hadoop cannot load", + Map(acct(Keys.KEY_PROVIDER) -> "", acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(kpe)), + c( + "sk-keyprovider-padded", + "sharedkey", + "keyprovider is read with get, which does not trim", + Map( + acct(Keys.KEY_PROVIDER) -> s" $SIMPLE_KEY_PROVIDER ", + acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(kpe)), + c( + "sk-keyprovider-custom", + "sharedkey", + "a custom KeyProvider supplies the key; no fs.azure.account.key needed", + Map(acct(Keys.KEY_PROVIDER) -> classOf[ParityKeyProvider].getName), + pins = allVersions(Pin.resolved)), + c( + "sk-keyprovider-custom-null", + "sharedkey", + "a KeyProvider returning null is reported against the account name", + Map(acct(Keys.KEY_PROVIDER) -> classOf[ParityNullKeyProvider].getName), + pins = allVersions(Pin.failed(ExceptionClasses.CONFIGURATION_PROPERTY_NOT_FOUND))), + c( + "sk-shell-echo", + "sharedkey", + "ShellDecryptionKeyProvider runs /bin/echo, which returns the envelope", + Map( + acct(Keys.KEY_PROVIDER) -> SHELL_KEY_PROVIDER, + acct(Keys.SHELL_KEY_PROVIDER_SCRIPT) -> "/bin/echo", + acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(Pin.resolved)), + c( + "sk-shell-script-unset", + "sharedkey", + "ShellDecryptionKeyProvider without a script", + Map(acct(Keys.KEY_PROVIDER) -> SHELL_KEY_PROVIDER, acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(kpe)), + c( + "sk-shell-script-missing", + "sharedkey", + "ShellDecryptionKeyProvider with a script that does not exist", + Map( + acct(Keys.KEY_PROVIDER) -> SHELL_KEY_PROVIDER, + acct(Keys.SHELL_KEY_PROVIDER_SCRIPT) -> "/nonexistent/comet-decrypt.sh", + acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(kpe)), + c( + "sk-account-beats-global", + "sharedkey", + "the account key wins over the global one", + Map(acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT, Keys.ACCOUNT_KEY -> KEY_GLOBAL)), + c( + "sk-container-ignored-with-account", + "sharedkey", + "SimpleKeyProvider never reads the container-scoped key", + Map(cont(Keys.ACCOUNT_KEY) -> KEY_CONTAINER, acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(Pin.resolved))) + validity ++ providers + } + + // ---- OAuth ---- + + private def oauthCases: Seq[ParityCase] = { + val tape = Pin.failed(ExceptionClasses.TOKEN_ACCESS_PROVIDER) + def missing(base: Map[String, String], key: String, id: String, pin: Map[String, Pin]) = + c( + s"oauth-$id", + "oauth", + s"${key.stripPrefix("fs.azure.account.oauth2.")} missing", + base - acct(key), + pins = pin) + + Seq( + // ClientCreds + c("oauth-clientcreds", "oauth", "ClientCreds, all keys set", clientCreds), + missing(clientCreds, Keys.CLIENT_ENDPOINT, "clientcreds-no-endpoint", allVersions(tape)), + missing(clientCreds, Keys.CLIENT_ID, "clientcreds-no-client-id", allVersions(tape)), + missing(clientCreds, Keys.CLIENT_SECRET, "clientcreds-no-secret", allVersions(tape)), + c( + "oauth-clientcreds-padded-endpoint", + "oauth", + "mandatory keys are forwarded untrimmed", + clientCreds + (acct(Keys.CLIENT_ENDPOINT) -> s" $ENDPOINT ")), + c( + "oauth-clientcreds-container-secret", + "oauth", + "container-scoped secret wins on 3.4.2+, is ignored before", + clientCreds + (cont(Keys.CLIENT_SECRET) -> "container-secret-value")), + c( + "oauth-clientcreds-global", + "oauth", + "ClientCreds configured globally", + clientCreds.map { case (k, v) => k.stripSuffix(s".$ACCOUNT") -> v }), + c( + "oauth-clientcreds-mixed-scopes", + "oauth", + "account auth type and provider, global endpoint and secret", + Map( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.CLIENT_CREDS, + Keys.CLIENT_ENDPOINT -> ENDPOINT, + acct(Keys.CLIENT_ID) -> CLIENT_ID, + Keys.CLIENT_SECRET -> CLIENT_SECRET)), + c( + "oauth-clientcreds-other-account-secret", + "oauth", + "another account's secret is never forwarded", + clientCreds + (acct(Keys.CLIENT_SECRET, "other.dfs.core.windows.net") -> "other-secret")), + // UserPassword + c("oauth-userpassword", "oauth", "UserPassword, all keys set", userPassword), + missing(userPassword, Keys.USER_NAME, "userpassword-no-user", allVersions(tape)), + missing(userPassword, Keys.USER_PASSWORD, "userpassword-no-password", allVersions(tape)), + missing(userPassword, Keys.CLIENT_ENDPOINT, "userpassword-no-endpoint", allVersions(tape)), + // MSI + c("oauth-msi", "oauth", "MSI with tenant and client id, defaults for the rest", msi), + c( + "oauth-msi-custom-endpoint-authority", + "oauth", + "MSI endpoint and authority are trimmed; the authority gets a trailing slash", + msi ++ Map( + acct(Keys.MSI_ENDPOINT) -> " http://msi.example.net/token ", + acct(Keys.MSI_AUTHORITY) -> " https://login.example.net ")), + c( + "oauth-msi-blank-endpoint", + "oauth", + "a blank MSI endpoint means the default", + msi + (acct(Keys.MSI_ENDPOINT) -> " ")), + c( + "oauth-msi-no-client-id", + "oauth", + "client id: mandatory on 3.3.4, optional from 3.4.1", + msi - acct(Keys.CLIENT_ID), + pins = from341(tape, Pin.resolved)), + c( + "oauth-msi-container-tenant", + "oauth", + "container-scoped MSI tenant on 3.4.2+", + msi + (cont(Keys.MSI_TENANT) -> "container-tenant")), + // RefreshToken + c("oauth-refreshtoken", "oauth", "RefreshToken, all keys set", refreshToken), + missing(refreshToken, Keys.REFRESH_TOKEN, "refreshtoken-no-token", allVersions(tape)), + missing(refreshToken, Keys.CLIENT_ID, "refreshtoken-no-client-id", allVersions(tape)), + c( + "oauth-refreshtoken-custom-endpoint", + "oauth", + "refresh token endpoint is trimmed", + refreshToken + (acct( + Keys.REFRESH_TOKEN_ENDPOINT) -> " https://login.example.net/token ")), + // WorkloadIdentity + c( + "oauth-wi", + "oauth", + "Workload Identity with the default token file", + workloadIdentity, + pins = fromWorkloadIdentity), + c( + "oauth-wi-no-tenant", + "oauth", + "Workload Identity tenant is mandatory", + workloadIdentity - acct(Keys.MSI_TENANT), + pins = from341(Pin.failed(ExceptionClasses.RUNTIME), tape)), + c( + "oauth-wi-no-client-id", + "oauth", + "Workload Identity client id is mandatory", + workloadIdentity - acct(Keys.CLIENT_ID), + pins = from341(Pin.failed(ExceptionClasses.RUNTIME), tape)), + c( + "oauth-wi-padded-token-file", + "oauth", + "token file is trimmed", + workloadIdentity + (acct(Keys.TOKEN_FILE) -> " /mnt/tokens/token "), + pins = fromWorkloadIdentity), + c( + "oauth-wi-container-token-file", + "oauth", + "container-scoped token file on 3.4.2+", + workloadIdentity + (cont(Keys.TOKEN_FILE) -> "/mnt/tokens/container-token"), + pins = fromWorkloadIdentity), + c( + "oauth-wi-blank-assertion-provider", + "oauth", + "a blank client assertion provider is ignored on 3.5.0", + workloadIdentity + (acct(Keys.CLIENT_ASSERTION_PROVIDER_TYPE) -> " "), + pins = fromWorkloadIdentity), + // provider class shapes + c( + "oauth-no-provider-type", + "oauth", + "OAuth without a provider class", + clientCreds - acct(Keys.OAUTH_PROVIDER_TYPE), + pins = allVersions(Pin.failed(ExceptionClasses.ILLEGAL_ARGUMENT))), + c( + "oauth-provider-not-built-in", + "oauth", + "an AccessTokenProvider outside the five built-ins is rejected by ==", + clientCreds + (acct(Keys.OAUTH_PROVIDER_TYPE) -> FOREIGN_ACCESS_TOKEN_PROVIDER), + pins = allVersions(Pin.failed(ExceptionClasses.ILLEGAL_ARGUMENT))), + c( + "oauth-provider-not-a-provider", + "oauth", + "a class that is not an AccessTokenProvider", + clientCreds + (acct(Keys.OAUTH_PROVIDER_TYPE) -> NOT_A_PROVIDER), + pins = allVersions(Pin.failed(ExceptionClasses.RUNTIME))), + c( + "oauth-provider-missing-class", + "oauth", + "a provider class that does not exist", + clientCreds + (acct(Keys.OAUTH_PROVIDER_TYPE) -> MISSING_CLASS), + pins = allVersions(Pin.failed(ExceptionClasses.RUNTIME))), + c( + "oauth-provider-empty", + "oauth", + "provider type '' is an unloadable class name", + clientCreds + (acct(Keys.OAUTH_PROVIDER_TYPE) -> ""), + pins = allVersions(Pin.failed(ExceptionClasses.RUNTIME))), + c( + "oauth-provider-padded", + "oauth", + "provider type is trimmed by Configuration.getClass", + clientCreds + (acct(Keys.OAUTH_PROVIDER_TYPE) -> s" ${Providers.CLIENT_CREDS} ")), + c( + "oauth-custom-not-adaptee", + "oauth", + "Custom with a class that is not a CustomTokenProviderAdaptee: declined, Hadoop fails", + Map( + acct(Keys.AUTH_TYPE) -> "Custom", + acct(Keys.OAUTH_PROVIDER_TYPE) -> FOREIGN_ACCESS_TOKEN_PROVIDER), + pins = allVersions(Pin.declined)), + c( + "oauth-custom-no-provider-type", + "oauth", + "Custom without a provider class: declined, Hadoop fails", + Map(acct(Keys.AUTH_TYPE) -> "Custom"), + pins = allVersions(Pin.declined))) + } + + // ---- auth-type guard: a global provider class needs a matching global auth type ---- + + private def guardCases: Seq[ParityCase] = { + def scopedKey(scope: String, key: String): String = if (scope == "global") key else acct(key) + val oauthValues = Map( + acct(Keys.CLIENT_ENDPOINT) -> ENDPOINT, + acct(Keys.CLIENT_ID) -> CLIENT_ID, + acct(Keys.CLIENT_SECRET) -> CLIENT_SECRET) + val unresolvedProviderClass = Pin.failed(ExceptionClasses.ILLEGAL_ARGUMENT) + val scopes = Seq( + ("global", "global"), + ("account", "account"), + ("account", "global"), + ("global", "account")) + scopes.flatMap { case (authScope, providerScope) => + val guardFails = authScope == "account" && providerScope == "global" + Seq( + c( + s"guard-oauth-$authScope-type-$providerScope-provider", + "guard", + s"$authScope auth type, $providerScope OAuth provider class", + oauthValues ++ Map( + scopedKey(authScope, Keys.AUTH_TYPE) -> "OAuth", + scopedKey(providerScope, Keys.OAUTH_PROVIDER_TYPE) -> Providers.CLIENT_CREDS), + pins = allVersions(if (guardFails) unresolvedProviderClass else Pin.resolved)), + c( + s"guard-sas-$authScope-type-$providerScope-provider", + "guard", + s"$authScope auth type, $providerScope SAS provider class", + Map( + scopedKey(authScope, Keys.AUTH_TYPE) -> "SAS", + scopedKey(providerScope, Keys.SAS_TOKEN_PROVIDER_TYPE) -> SAS_PROVIDER), + pins = if (guardFails) noSasSource else allVersions(Pin.declined))) + } + } + + // ---- SAS ---- + + private def sasCases: Seq[ParityCase] = { + val fixed = Map(acct(Keys.AUTH_TYPE) -> "SAS", acct(Keys.SAS_FIXED_TOKEN) -> FIXED_SAS) + Seq( + c("sas-fixed-only", "sas", "fixed token only", fixed, pins = fixedSasOnly), + c( + "sas-custom-only", + "sas", + "custom provider only", + sasCustom, + pins = allVersions(Pin.declined)), + c( + "sas-custom-and-fixed", + "sas", + "the custom provider wins over the fixed token", + sasCustom + (acct(Keys.SAS_FIXED_TOKEN) -> FIXED_SAS), + pins = allVersions(Pin.declined)), + c( + "sas-neither", + "sas", + "neither provider nor token", + Map(acct(Keys.AUTH_TYPE) -> "SAS"), + pins = noSasSource), + c( + "sas-blank-fixed", + "sas", + "a blank fixed token counts as unset", + fixed + (acct(Keys.SAS_FIXED_TOKEN) -> " "), + pins = noSasSource), + c( + "sas-question-mark", + "sas", + "a leading '?' is kept at init; the client strips it per request", + fixed + (acct(Keys.SAS_FIXED_TOKEN) -> s"?$FIXED_SAS"), + pins = fixedSasOnly), + c( + "sas-fixed-global", + "sas", + "global auth type and fixed token", + Map(Keys.AUTH_TYPE -> "SAS", Keys.SAS_FIXED_TOKEN -> FIXED_SAS), + pins = fixedSasOnly), + c( + "sas-provider-not-a-provider", + "sas", + "a class that is not a SASTokenProvider", + Map(acct(Keys.AUTH_TYPE) -> "SAS", acct(Keys.SAS_TOKEN_PROVIDER_TYPE) -> NOT_A_PROVIDER), + pins = allVersions(Pin.failed(ExceptionClasses.RUNTIME))), + c( + "sas-provider-missing-class", + "sas", + "a provider class that does not exist", + Map(acct(Keys.AUTH_TYPE) -> "SAS", acct(Keys.SAS_TOKEN_PROVIDER_TYPE) -> MISSING_CLASS), + pins = allVersions(Pin.failed(ExceptionClasses.RUNTIME)))) + } + + // ---- credential providers (JCEKS) and the clear-text fallback ---- + + private def credentialProviderCases(providerPath: String): Seq[ParityCase] = { + val provider = Map("hadoop.security.credential.provider.path" -> providerPath) + val noFallback = Map("hadoop.security.credential.clear-text-fallback" -> "false") + val kpe = Pin.failed(ExceptionClasses.KEY_PROVIDER) + Seq( + c( + "cp-jceks-account-key", + "credential-provider", + "the account key comes from the keystore; no clear-text key", + provider, + pins = allVersions(Pin.resolved)), + c( + "cp-jceks-beats-clear-text", + "credential-provider", + "the keystore is consulted before the clear-text value", + provider + (acct(Keys.ACCOUNT_KEY) -> KEY_GLOBAL), + pins = allVersions(Pin.resolved)), + c( + "cp-jceks-no-fallback", + "credential-provider", + "keystore with the clear-text fallback off", + provider ++ noFallback, + pins = allVersions(Pin.resolved)), + c( + "cp-jceks-client-secret", + "credential-provider", + "the OAuth client secret comes from the keystore", + provider ++ (clientCreds - acct(Keys.CLIENT_SECRET)), + pins = allVersions(Pin.resolved)), + c( + "cp-no-fallback-clear-text-only", + "credential-provider", + "fallback off, no keystore: a clear-text key is invisible, so nothing is configured", + noFallback + (acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(Pin.unconfigured)), + c( + "cp-no-fallback-with-auth-type", + "credential-provider", + "fallback off, no keystore, SharedKey named: the key provider finds nothing", + noFallback ++ Map( + acct(Keys.AUTH_TYPE) -> "SharedKey", + acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(kpe))) + } + + // ---- ${...} substitution ---- + + private def substitutionCases: Seq[ParityCase] = { + // Built by concatenation so scalac does not read them as interpolations missing their prefix. + val crossKey = "${" + "comet.parity.account.key}" + val envWithDefault = "${env." + UNSET_ENV_VAR + ":-fallback-client}" + val envHome = "${" + "env.HOME}" + Seq( + c( + "subst-cross-key", + "substitution", + "a reference to another key resolves through Configuration.get", + Map(acct(Keys.ACCOUNT_KEY) -> crossKey, "comet.parity.account.key" -> KEY_ENV), + pins = allVersions(Pin.resolved)), + c( + "subst-env-default", + "substitution", + "an environment reference with a default falls back when the variable is unset", + clientCreds + (acct(Keys.CLIENT_ID) -> envWithDefault), + pins = allVersions(Pin.resolved)), + c( + "subst-env-home", + "substitution", + "an environment reference reads the variable", + clientCreds + (acct(Keys.CLIENT_ID) -> envHome), + pins = allVersions(Pin.resolved), + liveValues = Map(Keys.CLIENT_ID -> sys.env.getOrElse("HOME", envHome)))) + } + + // ---- calibration: one case per version difference and per surprising Hadoop rule ---- + + private def calibrationCases: Seq[ParityCase] = { + val tape = Pin.failed(ExceptionClasses.TOKEN_ACCESS_PROVIDER) + val kpe = Pin.failed(ExceptionClasses.KEY_PROVIDER) + val badEnum = Pin.failed(ExceptionClasses.ILLEGAL_ARGUMENT) + val portAccount = s"$ACCOUNT:8080" + val prefixedEndpoint = "https://login.example.net/prefix/tenant-abc/oauth2/v2.0/token" + val assertionProvider = "com.example.comet.NeverLoadedAssertionProvider" + Seq( + c( + "cal-sas-guard-with-fixed-token", + "calibration", + "account SAS type + global SAS provider class: the guard drops the class, the token wins", + Map( + acct(Keys.AUTH_TYPE) -> "SAS", + Keys.SAS_TOKEN_PROVIDER_TYPE -> SAS_PROVIDER, + acct(Keys.SAS_FIXED_TOKEN) -> FIXED_SAS), + pins = fixedSasOnly), + c( + "cal-sas-guard-without-fixed-token", + "calibration", + "account SAS type + global SAS provider class and no token: nothing is left", + Map(acct(Keys.AUTH_TYPE) -> "SAS", Keys.SAS_TOKEN_PROVIDER_TYPE -> SAS_PROVIDER), + pins = noSasSource), + c( + "cal-sas-container-fixed-token", + "calibration", + "container-scoped fixed token: honoured from 3.4.2, invisible on 3.4.1", + Map(acct(Keys.AUTH_TYPE) -> "SAS", cont(Keys.SAS_FIXED_TOKEN) -> FIXED_SAS), + pins = pins( + Pin.failed(ExceptionClasses.TOKEN_ACCESS_PROVIDER), + Pin.failed(ExceptionClasses.SAS_TOKEN_PROVIDER), + Pin.resolved, + Pin.resolved)), + c( + "cal-sas-untrimmed-token", + "calibration", + "the fixed token is trimmed before use", + Map(acct(Keys.AUTH_TYPE) -> "SAS", acct(Keys.SAS_FIXED_TOKEN) -> s" $FIXED_SAS "), + pins = fixedSasOnly), + c( + "cal-endpoint-with-path-prefix", + "calibration", + "the client endpoint is forwarded verbatim, whatever its path shape", + clientCreds + (acct(Keys.CLIENT_ENDPOINT) -> prefixedEndpoint), + pins = allVersions(Pin.resolved)), + c( + "cal-port-key-matches", + "calibration", + "with a port in the URI, only the port-qualified key configures the account", + Map(acct(Keys.ACCOUNT_KEY, portAccount) -> KEY_PORT), + uri = uriFor(CONTAINER, portAccount), + pins = allVersions(Pin.resolved)), + c( + "cal-port-key-missing", + "calibration", + "with a port in the URI, the port-less key belongs to another account", + Map(acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + uri = uriFor(CONTAINER, portAccount), + pins = allVersions(Pin.unconfigured)), + c( + "cal-container-oauth-endpoint", + "calibration", + "container-scoped client endpoint: read from 3.4.2, missing before", + (clientCreds - acct(Keys.CLIENT_ENDPOINT)) + (cont(Keys.CLIENT_ENDPOINT) -> ENDPOINT), + pins = from342(tape, Pin.resolved)), + c( + "cal-container-account-key-only", + "calibration", + "a container-scoped account key alone: unread by SimpleKeyProvider on every version", + Map(cont(Keys.ACCOUNT_KEY) -> KEY_CONTAINER), + pins = from342(Pin.unconfigured, kpe)), + c( + "cal-global-enum-parsed-first", + "calibration", + "a bad global auth type fails even when the account one is valid", + clientCreds + (Keys.AUTH_TYPE -> "oauth"), + pins = allVersions(badEnum)), + c( + "cal-empty-mandatory-secret", + "calibration", + "'' passes the mandatory getter and is forwarded", + clientCreds + (acct(Keys.CLIENT_SECRET) -> ""), + pins = allVersions(Pin.resolved)), + c( + "cal-msi-no-tenant", + "calibration", + "MSI tenant: mandatory on 3.3.4, optional from 3.4.1", + msi - acct(Keys.MSI_TENANT), + pins = from341(tape, Pin.resolved)), + c( + "cal-msi-no-tenant-no-client-id", + "calibration", + "MSI with neither tenant nor client id", + msi - acct(Keys.MSI_TENANT) - acct(Keys.CLIENT_ID), + pins = from341(tape, Pin.resolved)), + c( + "cal-null-fsname-key", + "calibration", + "SimpleKeyProvider's 2-arg configuration probes the literal key.null. on 3.4.2+", + Map(s"${Keys.ACCOUNT_KEY}.null.$ACCOUNT" -> KEY_NULL_LITERAL), + pins = from342(Pin.unconfigured, Pin.resolved)), + c( + "cal-null-fsname-key-beats-account", + "calibration", + "key.null. is the first level on 3.4.2+, so it wins over the account key", + Map( + s"${Keys.ACCOUNT_KEY}.null.$ACCOUNT" -> KEY_NULL_LITERAL, + acct(Keys.ACCOUNT_KEY) -> KEY_ACCOUNT), + pins = allVersions(Pin.resolved)), + c( + "cal-base64-bad-length", + "calibration", + "Base64 validation: length % 4 != 0", + Map(acct(Keys.ACCOUNT_KEY) -> "c2VjcmV0c2V"), + pins = allVersions(kpe)), + c( + "cal-base64-non-ascii", + "calibration", + "Base64 validation: a char >= U+0080 escapes as ArrayIndexOutOfBoundsException", + Map(acct(Keys.ACCOUNT_KEY) -> "c2Vjcm\u00e90"), + pins = allVersions(Pin.failed(ExceptionClasses.ARRAY_INDEX))), + c( + "cal-empty-account-key", + "calibration", + "'' passes Base64 validation and fails in SharedKeyCredentials", + Map(acct(Keys.ACCOUNT_KEY) -> ""), + pins = allVersions(Pin.failed(ExceptionClasses.ILLEGAL_ARGUMENT))), + c( + "cal-wi-on-334", + "calibration", + "WorkloadIdentityTokenProvider does not exist on 3.3.4", + workloadIdentity, + pins = fromWorkloadIdentity), + c( + "cal-wi-client-assertion-provider", + "calibration", + "3.5.0 Workload Identity with a client assertion provider is declined, never loaded", + workloadIdentity + (acct(Keys.CLIENT_ASSERTION_PROVIDER_TYPE) -> s" $assertionProvider "), + pins = + pins(Pin.failed(ExceptionClasses.RUNTIME), Pin.resolved, Pin.resolved, Pin.declined)), + c( + "cal-userbound-sas", + "calibration", + "UserboundSASWithOAuth: an enum constant only from 3.5.0", + Map( + acct(Keys.AUTH_TYPE) -> "UserboundSASWithOAuth", + acct(Keys.SAS_TOKEN_PROVIDER_TYPE) -> SAS_PROVIDER), + pins = from350(badEnum, Pin.declined)), + c( + "cal-oauth-keys-without-auth-type", + "calibration", + "OAuth keys without an auth type are a SharedKey misconfiguration, not 'unconfigured'", + clientCreds - acct(Keys.AUTH_TYPE), + pins = allVersions(kpe))) + } +} + +/** A `KeyProvider` that supplies a fixed key; stands in for a vault-backed provider. */ +class ParityKeyProvider extends KeyProvider { + override def getStorageAccountKey(accountName: String, conf: Configuration): String = + AbfsAuthParityCases.KEY_CUSTOM +} + +/** A `KeyProvider` that finds nothing: Hadoop reports the account as a missing property. */ +class ParityNullKeyProvider extends KeyProvider { + override def getStorageAccountKey(accountName: String, conf: Configuration): String = null +} + +/** A `CustomTokenProviderAdaptee` that initializes fine, so a Custom FileSystem init succeeds. */ +class ParityCustomTokenAdaptee extends CustomTokenProviderAdaptee { + override def initialize(configuration: Configuration, accountName: String): Unit = () + override def getAccessToken(): String = "parity-token" + override def getExpiryTime(): Date = new Date(Long.MaxValue) +} + +/** A `SASTokenProvider` that initializes fine, so a SAS FileSystem init succeeds. */ +class ParitySasTokenProvider extends SASTokenProvider { + override def initialize(configuration: Configuration, accountName: String): Unit = () + override def getSASToken( + account: String, + fileSystem: String, + path: String, + operation: String): String = AbfsAuthParityCases.FIXED_SAS +} diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala new file mode 100644 index 00000000000..3d95864130a --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala @@ -0,0 +1,422 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.objectstore + +import java.io.InputStream +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} +import java.security.MessageDigest +import java.util.Comparator + +import scala.collection.immutable.SortedMap +import scala.collection.mutable +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.FileSystem +import org.apache.hadoop.security.alias.CredentialProviderFactory + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} + +import org.apache.comet.objectstore.AbfsAuthParityCases._ +import org.apache.comet.objectstore.AbfsAuthResolver._ +import org.apache.comet.objectstore.AbfsReflection.Handles + +/** + * Pins the resolver's behaviour per hadoop-azure version and cross-checks it against the real + * `AzureBlobFileSystem`, offline. + * + * Every [[AbfsAuthParityCases]] case runs through [[AbfsAuthResolver.resolve]] and is compared + * with `abfs-auth-parity/expected-.json`, where the version is the one the capability + * probes identify (never the Spark profile), and with the case's pins. Every case is then handed + * to `FileSystem.newInstance` with `fs.azure.account.hns.enabled=true`, which keeps 3.4.2+ + * initialization off the network. + * + * The FileSystem cross-check compares outcome classes, not values: `Resolved` needs a successful + * initialization, `Failed` an initialization that threw with the resolver's exception class in + * the cause chain, `Unconfigured` a `KeyProviderException`, and `Declined` either success or a + * failure while Hadoop loaded the provider the resolver left alone. Forwarded values are covered + * by the pins, the live-value checks and the digests in the recorded files, not by this check. + * + * To regenerate the expectation files, run the suite once per hadoop-azure line with + * `-Dcomet.abfs.parity.record=` (spark-3.5 for 3.3.4, spark-4.0 for 3.4.1, spark-4.1 for + * 3.4.2, spark-4.2 for 3.5.0) and copy the written files into + * `spark/src/test/resources/abfs-auth-parity/`. The pins and the FileSystem cross-check still + * assert while recording, so a recorded file never contradicts the hadoop-azure source. The file + * is written only when every case ran, so a filtered run cannot leave a partial file behind. + */ +class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterAll { + import AbfsAuthParitySuite._ + + private lazy val handles: Handles = AbfsReflection.handles(getClass.getClassLoader).get + private lazy val version: String = versionLabel(handles) + + private val recordDir: Option[Path] = Option(System.getProperty(RECORD_PROPERTY)) + // An undefined Maven property reaches the forked JVM as "null" or as the literal + // "${comet.abfs.parity.record}"; neither is a directory. + .map(_.trim) + .filter(p => p.nonEmpty && p != "null" && !p.startsWith("$")) + .map(p => Paths.get(p)) + + // The build points java.io.tmpdir under target/, which a fresh checkout may not have yet. + private val tempDir: Path = Files.createTempDirectory( + Files.createDirectories(Paths.get(System.getProperty("java.io.tmpdir"))), + "abfs-auth-parity") + private val keystore: Path = tempDir.resolve("credentials.jceks") + private val credentialProviderPath = "jceks://file" + keystore.toUri.getPath + + private lazy val cases: Seq[ParityCase] = AbfsAuthParityCases.cases(credentialProviderPath) + private lazy val expected: Map[String, Recorded] = + if (recordDir.isDefined) Map.empty else loadExpected(version) + private val recorded = mutable.Map[String, Recorded]() + + override def beforeAll(): Unit = { + super.beforeAll() + val conf = new Configuration() + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialProviderPath) + val provider = CredentialProviderFactory.getProviders(conf).get(0) + credentialEntries.foreach { case (alias, value) => + provider.createCredentialEntry(alias, value.toCharArray) + } + provider.flush() + } + + override def afterAll(): Unit = { + try { + recordDir.foreach { dir => + require( + recorded.size == cases.size, + s"only ${recorded.size} of ${cases.size} cases ran; not writing a partial expectation " + + s"file for hadoop-azure $version") + Files.createDirectories(dir) + val file = dir.resolve(s"expected-$version.json") + Files.write( + file, + renderExpected(version, recorded.toMap).getBytes(StandardCharsets.UTF_8)) + announce(s"AbfsAuthParitySuite recorded ${recorded.size} cases to $file") + } + } finally { + deleteRecursively(tempDir) + super.afterAll() + } + } + + // scalastyle:off println + private def announce(message: String): Unit = println(message) + // scalastyle:on println + + private def hadoopConf(c: ParityCase): Configuration = { + val conf = new Configuration() + c.conf.foreach { case (k, v) => conf.set(k, v) } + conf + } + + test("case ids are unique and every calibration case is pinned on all four versions") { + val ids = cases.map(_.id) + ids.distinct should have size ids.size.toLong + val unpinned = + cases.filter(_.group == "calibration").filterNot(_.pins.keySet == VERSIONS.toSet) + assert( + unpinned.isEmpty, + s"calibration cases without pins for every version: ${unpinned.map(_.id)}") + val unknownVersions = cases.flatMap(_.pins.keySet).distinct.filterNot(VERSIONS.contains) + assert(unknownVersions.isEmpty, s"pins for unknown versions: $unknownVersions") + assert(!sys.env.contains(UNSET_ENV_VAR), s"$UNSET_ENV_VAR must not be set in the environment") + info( + s"hadoop-azure $version, ${cases.size} cases, " + + cases + .groupBy(_.group) + .toSeq + .sortBy(_._1) + .map { case (g, cs) => s"$g=${cs.size}" } + .mkString(", ")) + } + + test("the expectation file matches the hadoop-azure version and the case list") { + assume(recordDir.isEmpty, "recording") + expected.keySet shouldBe cases.map(_.id).toSet + } + + cases.foreach { c => + test(s"${c.group}/${c.id}: ${c.note}") { + val outcome = resolve(hadoopConf(c), c.uri) + c.liveValues.foreach { case (key, value) => + withClue(s"live value of $key: ") { + outcome shouldBe a[Resolved] + outcome.asInstanceOf[Resolved].values.get(key) shouldBe Some(value) + } + } + val actual = Recorded.of(outcome, c.liveValues.keySet) + c.pins.get(version).foreach { pin => + withClue(s"pin for $version: ") { + actual.outcome shouldBe pin.outcome + actual.errorClass shouldBe pin.errorClass + } + } + recorded(c.id) = actual + if (recordDir.isEmpty) { + withClue(s"recorded expectation for hadoop-azure $version: ") { + actual shouldBe expected(c.id) + } + } + } + } + + test("FileSystem.newInstance agrees with the resolver on every case") { + val timings = mutable.ArrayBuffer[(String, Long)]() + val disagreements = cases.flatMap { c => + val outcome = resolve(hadoopConf(c), c.uri) + val started = System.nanoTime() + val fsResult = initializeFileSystem(c) + timings += c.id -> (System.nanoTime() - started) / 1000000L + checkAgreement(outcome, fsResult).map(reason => s"${c.id}: $reason") + } + info(s"${cases.size - disagreements.size} of ${cases.size} cases agree") + // A slow initialization would point at a network attempt, which this suite must never make. + val slowest = timings.sortBy(-_._2).take(3).map { case (id, ms) => s"$id ${ms}ms" } + info(s"slowest FileSystem initializations: ${slowest.mkString(", ")}") + assert(disagreements.isEmpty, disagreements.mkString("\n", "\n", "")) + } + + /** The real initialization, offline: HNS known, caches off, closed whatever happens. */ + private def initializeFileSystem(c: ParityCase): Either[Throwable, Unit] = { + val conf = hadoopConf(c) + if (Option(conf.get(HNS_ENABLED)).isEmpty) conf.set(HNS_ENABLED, "true") + conf.set("fs.abfs.impl.disable.cache", "true") + conf.set("fs.abfss.impl.disable.cache", "true") + var fs: Option[FileSystem] = None + try { + fs = Some(FileSystem.newInstance(c.uri, conf)) + Right(()) + } catch { + // The resolver reports LinkageErrors too, so the cross-check must see them. + case e @ (_: LinkageError | NonFatal(_)) => Left(e) + } finally { + fs.foreach(_.close()) + } + } + + private def checkAgreement(outcome: Outcome, fs: Either[Throwable, Unit]): Option[String] = { + def chain(e: Throwable): Seq[String] = causeChain(e).map(_.getClass.getName) + (outcome, fs) match { + case (_: Resolved, Right(())) => None + case (_: Resolved, Left(e)) => Some(s"resolver resolved, FileSystem threw ${chain(e)}") + case (Failed(cls, _), Right(())) => + Some(s"resolver failed with $cls, FileSystem initialized") + case (Failed(cls, _), Left(e)) => + if (chain(e).contains(cls)) None + else Some(s"resolver failed with $cls, FileSystem threw ${chain(e)}") + case (Unconfigured, Right(())) => Some("resolver unconfigured, FileSystem initialized") + case (Unconfigured, Left(e)) => + if (chain(e).contains(ExceptionClasses.KEY_PROVIDER)) None + else Some(s"resolver unconfigured, FileSystem threw ${chain(e)} not KeyProviderException") + case (_: Declined, Right(())) => None + case (_: Declined, Left(e)) => + // Hadoop instantiates the provider the resolver declined to; whatever that does is fine. + if (chain(e).exists(PROVIDER_FAILURES.contains)) None + else Some(s"resolver declined, FileSystem threw ${chain(e)} outside provider loading") + } + } +} + +object AbfsAuthParitySuite { + val RECORD_PROPERTY = "comet.abfs.parity.record" + val RESOURCE_DIR = "abfs-auth-parity" + private val HNS_ENABLED = "fs.azure.account.hns.enabled" + private val MAX_CAUSE_DEPTH = 8 + + private val PROVIDER_FAILURES: Set[String] = Set( + ExceptionClasses.TOKEN_ACCESS_PROVIDER, + ExceptionClasses.SAS_TOKEN_PROVIDER, + ExceptionClasses.ILLEGAL_ARGUMENT) + + /** The forwarded values whose recorded form is a SHA-256 digest. */ + private val SECRET_KEYS: Set[String] = Set( + Keys.ACCOUNT_KEY, + Keys.CLIENT_SECRET, + Keys.USER_PASSWORD, + Keys.REFRESH_TOKEN, + Keys.SAS_FIXED_TOKEN) + private val ENVIRONMENT_PLACEHOLDER = "" + + /** The hadoop-azure line on the classpath, from what it can do rather than from a version. */ + def versionLabel(handles: Handles): String = + if (handles.hasClientAssertionProvider) "3.5.0" + else if (handles.hasContainerScopedConfiguration) "3.4.2" + else if (handles.hasFixedSasTokenProvider) "3.4.1" + else "3.3.4" + + /** + * One case's expectation; `values` holds digests for secrets and a placeholder for live values. + */ + final case class Recorded( + outcome: String, + authType: Option[String], + providerClass: Option[String], + assertionProviderClass: Option[String], + errorClass: Option[String], + values: SortedMap[String, String]) + + object Recorded { + def of(outcome: Outcome, liveKeys: Set[String]): Recorded = outcome match { + case Resolved(authType, providerClass, values) => + val normalized = values.map { case (k, v) => + val recorded = + if (liveKeys.contains(k)) ENVIRONMENT_PLACEHOLDER + else if (SECRET_KEYS.contains(k)) sha256(v) + else v + k -> recorded + } + Recorded( + Outcomes.RESOLVED, + Some(authType), + providerClass, + None, + None, + SortedMap(normalized.toSeq: _*)) + case Unconfigured => + Recorded(Outcomes.UNCONFIGURED, None, None, None, None, SortedMap.empty) + case Declined(authType, providerClass, assertionProviderClass) => + Recorded( + Outcomes.DECLINED, + Some(authType), + providerClass, + assertionProviderClass, + None, + SortedMap.empty) + case Failed(exceptionClass, _) => + Recorded(Outcomes.FAILED, None, None, None, Some(exceptionClass), SortedMap.empty) + } + } + + private def sha256(value: String): String = { + val digest = + MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)) + "sha256:" + digest.map(b => f"$b%02x").mkString + } + + private def causeChain(e: Throwable): Seq[Throwable] = { + val chain = mutable.ArrayBuffer[Throwable](e) + var cause = e.getCause + while (cause != null && chain.length < MAX_CAUSE_DEPTH && !chain.contains(cause)) { + chain += cause + cause = cause.getCause + } + chain.toSeq + } + + private def loadExpected(version: String): Map[String, Recorded] = { + val resource = s"/$RESOURCE_DIR/expected-$version.json" + val stream: InputStream = getClass.getResourceAsStream(resource) + require( + stream != null, + s"$resource is missing; record it with -D$RECORD_PROPERTY= on a classpath with " + + s"hadoop-azure $version") + try { + val root = new ObjectMapper().readTree(stream) + require( + root.get("hadoopAzureVersion").asText == version, + s"$resource records ${root.get("hadoopAzureVersion").asText}, classpath has $version") + val cases = root.get("cases") + // fieldNames() rather than fields(): the latter is deprecated on the Jackson Spark 4 ships + // and its replacement is missing on the one Spark 3 ships. + cases.fieldNames().asScala.map(id => id -> parseRecorded(cases.get(id))).toMap + } finally { + stream.close() + } + } + + private def parseRecorded(node: JsonNode): Recorded = { + def optional(field: String): Option[String] = Option(node.get(field)).map(_.asText) + val values = Option(node.get("values")) + .map(v => v.fieldNames().asScala.map(k => k -> v.get(k).asText).toSeq) + .getOrElse(Seq.empty) + Recorded( + node.get("outcome").asText, + optional("authType"), + optional("providerClass"), + optional("assertionProviderClass"), + optional("errorClass"), + SortedMap(values: _*)) + } + + /** Sorted keys and a fixed layout, so a regeneration diff shows only behaviour changes. */ + private def renderExpected(version: String, recorded: Map[String, Recorded]): String = { + val cases: SortedMap[String, Any] = SortedMap(recorded.toSeq: _*).map { case (id, r) => + val fields: Seq[(String, Any)] = Seq( + Some("outcome" -> r.outcome), + r.authType.map("authType" -> _), + r.providerClass.map("providerClass" -> _), + r.assertionProviderClass.map("assertionProviderClass" -> _), + r.errorClass.map("errorClass" -> _), + Some("values" -> r.values)).flatten + id -> SortedMap[String, Any](fields: _*) + } + val root = SortedMap[String, Any]("cases" -> cases, "hadoopAzureVersion" -> version) + val out = new StringBuilder + renderJson(root, 0, out) + out.append('\n').toString + } + + private def renderJson(value: Any, indent: Int, out: StringBuilder): Unit = value match { + case s: String => out.append(quote(s)) + case m: scala.collection.Map[_, _] if m.isEmpty => out.append("{}") + case m: scala.collection.Map[_, _] => + val pad = " " * (indent + 1) + out.append("{\n") + val entries = m.toSeq.map { case (k, v) => k.toString -> v }.sortBy(_._1) + entries.zipWithIndex.foreach { case ((k, v), i) => + out.append(pad).append(quote(k)).append(": ") + renderJson(v, indent + 1, out) + if (i < entries.size - 1) out.append(',') + out.append('\n') + } + out.append(" " * indent).append('}') + case other => throw new IllegalArgumentException(s"unsupported JSON value: $other") + } + + private def quote(s: String): String = { + val out = new StringBuilder("\"") + s.foreach { + case '"' => out.append("\\\"") + case '\\' => out.append("\\\\") + case '\n' => out.append("\\n") + case '\r' => out.append("\\r") + case '\t' => out.append("\\t") + case ch if ch < 0x20 || ch > 0x7e => out.append('\\').append('u').append(f"${ch.toInt}%04x") + case ch => out.append(ch) + } + out.append('"').toString + } + + private def deleteRecursively(dir: Path): Unit = { + if (Files.exists(dir)) { + Files.walk(dir).sorted(Comparator.reverseOrder[Path]()).forEach { (p: Path) => + Files.deleteIfExists(p) + () + } + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala new file mode 100644 index 00000000000..4d69c2e01a3 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala @@ -0,0 +1,703 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.objectstore + +import java.net.URI + +import scala.collection.mutable.ArrayBuffer +import scala.util.{Failure, Success, Try} + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration +import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider +import org.apache.hadoop.fs.azurebfs.oauth2.{AccessTokenProvider, AzureADToken} +import org.apache.hadoop.fs.azurebfs.services.KeyProvider + +import org.apache.comet.objectstore.AbfsAuthResolver._ +import org.apache.comet.objectstore.AbfsReflection.Handles + +/** + * Runs the resolver against the hadoop-azure on the test classpath (3.3.4 on the spark-3.x + * profiles, 3.4.1, 3.4.2 and 3.5.0 on spark-4.0, 4.1 and 4.2). Cases that depend on a feature one + * line lacks are gated on the capability probes rather than on a version number. + */ +class AbfsAuthResolverSuite extends AnyFunSuite with Matchers { + + private val account = "myacct.dfs.core.windows.net" + private val container = "data" + private val uri = new URI(s"abfss://$container@$account/path/file.parquet") + + private val keyProviderException = + "org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException" + private val invalidUriException = + "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException" + private val invalidUriAuthorityException = + "org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException" + private val tokenAccessProviderException = + "org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException" + + private lazy val handles: Handles = AbfsReflection.handles(getClass.getClassLoader).get + + private def acct(key: String): String = s"$key.$account" + + private def cont(key: String): String = s"$key.$container.$account" + + private def conf(pairs: (String, String)*): Configuration = { + val hadoopConf = new Configuration() + pairs.foreach { case (k, v) => hadoopConf.set(k, v) } + hadoopConf + } + + private def resolveWith(pairs: (String, String)*): Outcome = resolve(conf(pairs: _*), uri) + + /** + * `key` set to a `${...}` cycle through `other`, which `Configuration#get` cannot expand and + * throws on. Built by concatenation so scalac does not read it as a missing interpolator. + */ + private def cyclic(key: String, other: String): Seq[(String, String)] = + Seq(key -> ("${" + other + "}"), other -> ("${" + key + "}")) + + private def assertNoAzureKeys(options: Map[String, String]): Unit = { + val leaked = options.keys.filter(_.startsWith("fs.")) + assert(leaked.isEmpty, s"fs.* keys forwarded: $leaked") + } + + // ---- authorityParts ---- + + test("authorityParts - raw authority split once at the first '@', port and case kept") { + authorityParts(new URI("abfss://data@myacct.dfs.core.windows.net:8080/p")) shouldBe + Right(("data", "myacct.dfs.core.windows.net:8080")) + authorityParts(new URI("abfss://Data@MyAcct.DFS.core.windows.net/p")) shouldBe + Right(("Data", "MyAcct.DFS.core.windows.net")) + authorityParts(new URI("abfss://my%20data@myacct.dfs.core.windows.net/p")) shouldBe + Right(("my%20data", "myacct.dfs.core.windows.net")) + authorityParts(new URI("abfss://data@user@myacct.dfs.core.windows.net/p")) shouldBe + Right(("data", "user@myacct.dfs.core.windows.net")) + } + + test("authorityParts - no '@', empty container and null authority fail as Hadoop does") { + val noAt = "abfss://myacct.dfs.core.windows.net/p" + authorityParts(new URI(noAt)) shouldBe + Left(Failed(invalidUriAuthorityException, s"$noAt has invalid authority.")) + + val emptyContainer = "abfss://@myacct.dfs.core.windows.net/p" + authorityParts(new URI(emptyContainer)) shouldBe Left( + Failed( + invalidUriException, + s"Invalid URI '$emptyContainer' has a malformed authority, expected container name. " + + "Authority takes the form abfs://[@]")) + + val nullAuthority = "abfss:///p" + authorityParts(new URI(nullAuthority)) shouldBe + Left(Failed(invalidUriAuthorityException, s"$nullAuthority has invalid authority.")) + } + + // ---- reflection ---- + + test("handles - the constructor choice matches the classpath's AbfsConfiguration") { + // Hadoop 3.4.2 added `containerConf` together with the constructor that takes the container. + val hasContainerConf = + Try(classOf[AbfsConfiguration].getMethod("containerConf", classOf[String])).isSuccess + handles.hasContainerScopedConfiguration shouldBe hasContainerConf + info( + s"container-scoped=${handles.hasContainerScopedConfiguration} " + + s"fixedSas=${handles.hasFixedSasTokenProvider} " + + s"clientAssertion=${handles.hasClientAssertionProvider}") + } + + test("handles - a failed lookup is not cached; the same class yields the same handles") { + val loader = new FlippingLoader(getClass.getClassLoader) + AbfsReflection.handles(loader).isFailure shouldBe true + AbfsReflection.handles(loader).isFailure shouldBe true + loader.blocked = false + // The delegating loader hands back the parent's AbfsConfiguration class, so the cache entry + // is the parent's too. + AbfsReflection.handles(loader) shouldBe Success(handles) + } + + // ---- Unconfigured ---- + + test("Unconfigured - no auth-bearing key for the account, even with other Azure keys set") { + resolveWith() shouldBe Unconfigured + resolveWith("fs.azure.account.hns.enabled" -> "true") shouldBe Unconfigured + // Another account's key does not configure this one. + resolveWith("fs.azure.account.key.other.dfs.core.windows.net" -> "c2VjcmV0") shouldBe + Unconfigured + toOptions(Unconfigured) shouldBe Map(Markers.RESOLUTION -> Markers.NONE) + } + + test("not Unconfigured - OAuth keys without an auth type fail in Hadoop's SharedKey words") { + // Hadoop defaults to SharedKey, whose key provider finds no account key. The environment must + // not step in for a misconfigured account. + val outcome = resolveWith(acct(Keys.CLIENT_ID) -> "client-123") + outcome shouldBe a[Failed] + outcome.asInstanceOf[Failed].exceptionClass shouldBe keyProviderException + assertNoAzureKeys(toOptions(outcome)) + } + + test("not Unconfigured - a fixed SAS token without an auth type, on every hadoop-azure") { + val outcome = resolveWith(acct(Keys.SAS_FIXED_TOKEN) -> "sv=2020-08-04&sig=fixed-signature") + outcome shouldBe a[Failed] + outcome.asInstanceOf[Failed].exceptionClass shouldBe keyProviderException + assertNoAzureKeys(toOptions(outcome)) + } + + test("not Unconfigured - every auth-bearing key set alone hands the decision to Hadoop") { + val plainValues = Map( + Keys.AUTH_TYPE -> "SharedKey", + Keys.KEY_PROVIDER -> "org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider", + Keys.SHELL_KEY_PROVIDER_SCRIPT -> "/bin/echo", + Keys.OAUTH_PROVIDER_TYPE -> Providers.CLIENT_CREDS, + Keys.SAS_TOKEN_PROVIDER_TYPE -> classOf[NeverInstantiatedSasTokenProvider].getName) + plainValues.keySet shouldBe plainAuthKeys.toSet + val cases = plainValues.toSeq ++ passwordAuthKeys.map(_ -> "some-value") + cases.foreach { case (key, value) => + withClue(s"$key alone: ") { + val outcome = resolveWith(acct(key) -> value) + outcome should not be Unconfigured + outcome shouldBe a[Failed] + assertNoAzureKeys(toOptions(outcome)) + } + } + } + + // ---- Declined ---- + + test("Declined - Custom auth type is forwarded by name, never instantiated") { + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "Custom", + acct(Keys.OAUTH_PROVIDER_TYPE) -> classOf[NeverInstantiatedAccessTokenProvider].getName) + outcome shouldBe Declined("Custom", None) + toOptions(outcome) shouldBe + Map(Markers.RESOLUTION -> Markers.DECLINED, Markers.AUTH_TYPE -> "Custom") + } + + test("Declined - a custom SAS token provider class is named, never constructed") { + val providerClass = classOf[NeverInstantiatedSasTokenProvider].getName + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "SAS", + acct(Keys.SAS_TOKEN_PROVIDER_TYPE) -> providerClass, + acct(Keys.SAS_FIXED_TOKEN) -> "sv=2020-08-04&sig=ignored-when-a-class-is-set") + outcome shouldBe Declined("SAS", Some(providerClass)) + val options = toOptions(outcome) + options(Markers.SAS_PROVIDER_CLASS) shouldBe providerClass + assertNoAzureKeys(options) + } + + test("Failed - an AccessTokenProvider that is not one of the five built-ins fails in Hadoop") { + // Hadoop matches the class by `==` and rejects anything else without constructing it. + val providerClass = classOf[NeverInstantiatedAccessTokenProvider].getName + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> providerClass, + acct(Keys.CLIENT_ID) -> "client-123") + outcome shouldBe Failed( + classOf[IllegalArgumentException].getName, + s"java.lang.IllegalArgumentException: Failed to initialize class $providerClass") + assertNoAzureKeys(toOptions(outcome)) + } + + test("Declined - Workload Identity with a client assertion provider (hadoop-azure 3.5.0+)") { + assume(handles.hasClientAssertionProvider, "no ClientAssertionProvider on this classpath") + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.WORKLOAD_IDENTITY, + acct(Keys.MSI_TENANT) -> "tenant-abc", + acct(Keys.CLIENT_ID) -> "client-123", + acct(Keys.CLIENT_ASSERTION_PROVIDER_TYPE) -> " com.example.NeverLoadedAssertionProvider ") + outcome shouldBe Declined( + "OAuth", + Some(Providers.WORKLOAD_IDENTITY), + Some("com.example.NeverLoadedAssertionProvider")) + val options = toOptions(outcome) + options(Markers.OAUTH_PROVIDER_CLASS) shouldBe Providers.WORKLOAD_IDENTITY + options(Markers.OAUTH_ASSERTION_PROVIDER_CLASS) shouldBe + "com.example.NeverLoadedAssertionProvider" + assertNoAzureKeys(options) + } + + test("client assertion key is ignored where Hadoop ignores it (3.4.1 and 3.4.2)") { + assume( + handles.hasFixedSasTokenProvider && !handles.hasClientAssertionProvider, + "only on a hadoop-azure with Workload Identity but without ClientAssertionProvider") + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.WORKLOAD_IDENTITY, + acct(Keys.MSI_TENANT) -> "tenant-abc", + acct(Keys.CLIENT_ID) -> "client-123", + acct(Keys.CLIENT_ASSERTION_PROVIDER_TYPE) -> "com.example.NeverLoadedAssertionProvider") + outcome shouldBe a[Resolved] + outcome.asInstanceOf[Resolved].providerClass shouldBe Some(Providers.WORKLOAD_IDENTITY) + } + + // ---- Failed ---- + + test("Failed - a bad Base64 account key carries Hadoop's class and a redacted message") { + val rawKey = "this-is-not-base64!!" + val outcome = resolveWith(acct(Keys.ACCOUNT_KEY) -> rawKey) + outcome shouldBe a[Failed] + val failed = outcome.asInstanceOf[Failed] + failed.exceptionClass shouldBe keyProviderException + failed.message should include("KeyProviderException") + failed.message should include("driver log") + failed.message should not include rawKey + val options = toOptions(outcome) + options(Markers.RESOLUTION) shouldBe Markers.ERROR + options(Markers.ERROR_CLASS) shouldBe keyProviderException + options.values.foreach(v => v should not include rawKey) + assertNoAzureKeys(options) + } + + test("Failed - a KeyProvider's own IllegalArgumentException is reported by class only") { + // Hadoop's IllegalArgumentExceptions name keys and classes; a user KeyProvider's may quote a + // secret the resolver never read, so only its origin decides whether the text is forwarded. + val outcome = + resolveWith(acct(Keys.KEY_PROVIDER) -> classOf[SecretLeakingKeyProvider].getName) + outcome shouldBe Failed( + classOf[IllegalArgumentException].getName, + "java.lang.IllegalArgumentException; the full message is in the Spark driver log") + val options = toOptions(outcome) + options(Markers.ERROR_CLASS) shouldBe classOf[IllegalArgumentException].getName + options.values.foreach(v => v should not include SecretLeakingKeyProvider.marker) + assertNoAzureKeys(options) + } + + test("Failed - a credential the resolution never read is still redacted") { + // A client secret pasted into the auth type: Hadoop's enum error quotes it, and resolution + // stops at the auth type, so only the failure path reads the secret it must hide. + val secret = "oauth-client-secret-4f9a2c" + val outcome = resolveWith(acct(Keys.AUTH_TYPE) -> secret, acct(Keys.CLIENT_SECRET) -> secret) + outcome shouldBe a[Failed] + val failed = outcome.asInstanceOf[Failed] + failed.exceptionClass shouldBe classOf[IllegalArgumentException].getName + failed.message should include("No enum constant") + failed.message should include("") + failed.message should not include secret + } + + test("Failed - an unknown auth type value is Hadoop's enum error") { + val outcome = resolveWith(acct(Keys.AUTH_TYPE) -> "oauth") + outcome shouldBe a[Failed] + val failed = outcome.asInstanceOf[Failed] + failed.exceptionClass shouldBe classOf[IllegalArgumentException].getName + failed.message should include("No enum constant") + failed.message should include("oauth") + } + + test("Failed - account-specific OAuth with only a global provider class (auth-type guard)") { + // getTokenProviderClass reads the global provider class only when the GLOBAL auth type + // matches, so Hadoop sees no class and fails to initialize the provider. + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + Keys.OAUTH_PROVIDER_TYPE -> Providers.CLIENT_CREDS, + acct(Keys.CLIENT_ENDPOINT) -> "https://login.microsoftonline.com/tenant/oauth2/token", + acct(Keys.CLIENT_ID) -> "client-123", + acct(Keys.CLIENT_SECRET) -> "client-secret-value") + outcome shouldBe Failed( + classOf[IllegalArgumentException].getName, + "java.lang.IllegalArgumentException: Failed to initialize null") + } + + test("Failed - a missing mandatory OAuth key surfaces Hadoop's cause chain") { + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.CLIENT_CREDS, + acct(Keys.CLIENT_ENDPOINT) -> "https://login.microsoftonline.com/tenant/oauth2/token", + acct(Keys.CLIENT_ID) -> "client-123") + outcome shouldBe a[Failed] + val failed = outcome.asInstanceOf[Failed] + failed.exceptionClass shouldBe tokenAccessProviderException + failed.message should include(s"Configuration property ${Keys.CLIENT_SECRET} not found.") + assertNoAzureKeys(toOptions(outcome)) + } + + test("Failed - SharedKey with an account name that is not fully qualified") { + val shortUri = new URI("abfss://data@myacct/path") + val outcome = resolve(conf("fs.azure.account.key.myacct" -> "c2VjcmV0"), shortUri) + outcome shouldBe + Failed(invalidUriException, s"Invalid URI $shortUri - account name is not fully qualified.") + } + + test("Failed - a LinkageError from the reflection layer is an error marker, not a crash") { + val outcome = resolve(conf(acct(Keys.ACCOUNT_KEY) -> "c2VjcmV0"), uri, Success(LinkageStub)) + outcome shouldBe a[Failed] + outcome.asInstanceOf[Failed].exceptionClass shouldBe classOf[NoSuchMethodError].getName + assertNoAzureKeys(toOptions(outcome)) + } + + test("Failed - hadoop-azure missing from the classpath") { + val missing = new ClassNotFoundException(AbfsReflection.ClassNames.ABFS_CONFIGURATION) + val outcome = resolve(conf(acct(Keys.ACCOUNT_KEY) -> "c2VjcmV0"), uri, Failure(missing)) + outcome shouldBe a[Failed] + val failed = outcome.asInstanceOf[Failed] + failed.exceptionClass shouldBe classOf[ClassNotFoundException].getName + failed.message should include("hadoop-azure is not on the driver classpath") + failed.message should include(AbfsReflection.ClassNames.ABFS_CONFIGURATION) + } + + test("redact - hides quoted keys and long secrets, leaves short values alone") { + val message = "Failure for acct key =\"c2VjcmV0c2VjcmV0\": token=the-secret-value port=8080" + redact(message, Seq("the-secret-value", "8080")) shouldBe + "Failure for acct key =\"\": token= port=8080" + } + + test("redact - a padded secret is hidden in its trimmed form too") { + val padded = " the-padded-secret " + redact("provider said the-padded-secret was rejected", Seq(padded, padded.trim)) shouldBe + "provider said was rejected" + } + + // ---- Resolved ---- + + test("Resolved - SharedKey from the account-scoped key, forwarded under the global name") { + val outcome = resolveWith(acct(Keys.ACCOUNT_KEY) -> "c2VjcmV0") + outcome shouldBe Resolved("SharedKey", None, Map(Keys.ACCOUNT_KEY -> "c2VjcmV0")) + toOptions(outcome) shouldBe Map( + Markers.RESOLUTION -> Markers.RESOLVED, + Markers.AUTH_TYPE -> "SharedKey", + Keys.ACCOUNT_KEY -> "c2VjcmV0") + } + + test("Resolved - SharedKey reads no more credentials than Hadoop needs for the key") { + // Every getPassword can load a credential provider's keystore, so the resolver must not + // read the credentials of mechanisms Hadoop did not select. + def sharedKeyConf(): CountingConfiguration = { + val hadoopConf = new CountingConfiguration + hadoopConf.set(Keys.AUTH_TYPE, "SharedKey") + hadoopConf.set(acct(Keys.ACCOUNT_KEY), "c2VjcmV0") + hadoopConf + } + val direct = sharedKeyConf() + handles.getStorageAccountKey( + handles.newAbfsConfiguration(direct, account, container, uri)) shouldBe "c2VjcmV0" + direct.passwordReads should not be empty + + val viaResolver = sharedKeyConf() + resolve(viaResolver, uri) shouldBe + Resolved("SharedKey", None, Map(Keys.ACCOUNT_KEY -> "c2VjcmV0")) + val reads = s"Hadoop read ${direct.passwordReads}, the resolver ${viaResolver.passwordReads}" + withClue(reads) { + viaResolver.passwordReads.size should be <= direct.passwordReads.size + } + } + + test("Resolved - SharedKey with an unreadable OAuth client secret it never reads") { + val outcome = resolveWith( + Seq(acct(Keys.ACCOUNT_KEY) -> "c2VjcmV0") ++ + cyclic(Keys.CLIENT_SECRET, "unused.secret"): _*) + outcome shouldBe Resolved("SharedKey", None, Map(Keys.ACCOUNT_KEY -> "c2VjcmV0")) + } + + test("Resolved - SharedKey with an unreadable OAuth provider type it never reads") { + val outcome = resolveWith( + Seq(acct(Keys.ACCOUNT_KEY) -> "c2VjcmV0") ++ + cyclic(Keys.OAUTH_PROVIDER_TYPE, "unused.provider"): _*) + outcome shouldBe Resolved("SharedKey", None, Map(Keys.ACCOUNT_KEY -> "c2VjcmV0")) + } + + test("not Unconfigured - an unreadable key alone fails in Hadoop's SharedKey words") { + Seq(Keys.CLIENT_SECRET, Keys.OAUTH_PROVIDER_TYPE).foreach { key => + withClue(s"$key alone: ") { + val outcome = resolveWith(cyclic(key, "unused.value"): _*) + outcome shouldBe a[Failed] + outcome.asInstanceOf[Failed].exceptionClass shouldBe keyProviderException + } + } + } + + test("Failed - an unreadable account key fails whether or not SharedKey is named") { + Seq(Seq.empty[(String, String)], Seq(acct(Keys.AUTH_TYPE) -> "SharedKey")).foreach { + authType => + withClue(s"auth type $authType: ") { + val outcome = resolveWith(authType ++ cyclic(acct(Keys.ACCOUNT_KEY), "cycle.key"): _*) + outcome shouldBe a[Failed] + assertNoAzureKeys(toOptions(outcome)) + } + } + } + + test("Resolved - SharedKey from the global key, with a port-scoped account") { + resolveWith(Keys.ACCOUNT_KEY -> "Z2xvYmFs") shouldBe + Resolved("SharedKey", None, Map(Keys.ACCOUNT_KEY -> "Z2xvYmFs")) + + val portUri = new URI("abfss://data@myacct.dfs.core.windows.net:8080/path") + val portConf = conf( + "fs.azure.account.key.myacct.dfs.core.windows.net:8080" -> "cG9ydA==", + acct(Keys.ACCOUNT_KEY) -> "bm9wb3J0") + resolve(portConf, portUri) shouldBe + Resolved("SharedKey", None, Map(Keys.ACCOUNT_KEY -> "cG9ydA==")) + } + + test("Resolved - mixed-case account names are matched exactly, as Hadoop does") { + val mixedUri = new URI("abfss://data@MyAcct.dfs.core.windows.net/path") + resolve( + conf("fs.azure.account.key.MyAcct.dfs.core.windows.net" -> "bWl4ZWQ="), + mixedUri) shouldBe + Resolved("SharedKey", None, Map(Keys.ACCOUNT_KEY -> "bWl4ZWQ=")) + // The lower-case key belongs to a different account name. + resolve( + conf("fs.azure.account.key.myacct.dfs.core.windows.net" -> "bWl4ZWQ="), + mixedUri) shouldBe + Unconfigured + } + + test("Resolved - ClientCreds forwards only its keys, never another account's") { + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.CLIENT_CREDS, + acct(Keys.CLIENT_ENDPOINT) -> "https://login.microsoftonline.com/tenant/oauth2/token", + acct(Keys.CLIENT_ID) -> "client-123", + acct(Keys.CLIENT_SECRET) -> "client-secret-value", + // Keys MSI would read, and another account's secret: none of them may travel. + acct(Keys.MSI_TENANT) -> "tenant-abc", + "fs.azure.account.oauth2.client.secret.other.dfs.core.windows.net" -> "other-secret-value") + outcome shouldBe Resolved( + "OAuth", + Some(Providers.CLIENT_CREDS), + Map( + Keys.CLIENT_ENDPOINT -> "https://login.microsoftonline.com/tenant/oauth2/token", + Keys.CLIENT_ID -> "client-123", + Keys.CLIENT_SECRET -> "client-secret-value")) + val options = toOptions(outcome) + options(Markers.OAUTH_PROVIDER_CLASS) shouldBe Providers.CLIENT_CREDS + options.values.foreach(v => v should not include "other-secret-value") + options.keys.filter(_.startsWith("fs.")) should contain theSameElementsAs + Seq(Keys.CLIENT_ENDPOINT, Keys.CLIENT_ID, Keys.CLIENT_SECRET) + } + + test("Resolved - global OAuth configuration applies when the global auth type matches") { + val globalConf = conf( + Keys.AUTH_TYPE -> "OAuth", + Keys.OAUTH_PROVIDER_TYPE -> Providers.CLIENT_CREDS, + Keys.CLIENT_ENDPOINT -> "https://login.microsoftonline.com/tenant/oauth2/token", + Keys.CLIENT_ID -> "global-client", + // A variable reference resolves through Configuration#get, as it does for Hadoop. Built by + // concatenation so scalac does not read it as a string missing its interpolator. + "my.client.secret" -> "substituted-secret", + Keys.CLIENT_SECRET -> ("${" + "my.client.secret}")) + resolve(globalConf, uri) shouldBe Resolved( + "OAuth", + Some(Providers.CLIENT_CREDS), + Map( + Keys.CLIENT_ENDPOINT -> "https://login.microsoftonline.com/tenant/oauth2/token", + Keys.CLIENT_ID -> "global-client", + Keys.CLIENT_SECRET -> "substituted-secret")) + } + + test("Resolved - MSI applies Hadoop's defaults, trimming and the authority slash") { + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.MSI, + acct(Keys.MSI_TENANT) -> "tenant-abc", + acct(Keys.CLIENT_ID) -> "client-123", + acct(Keys.MSI_AUTHORITY) -> " https://login.example.net ") + outcome shouldBe Resolved( + "OAuth", + Some(Providers.MSI), + Map( + Keys.MSI_ENDPOINT -> "http://169.254.169.254/metadata/identity/oauth2/token", + Keys.MSI_TENANT -> "tenant-abc", + Keys.CLIENT_ID -> "client-123", + Keys.MSI_AUTHORITY -> "https://login.example.net/")) + } + + test("Resolved - MSI without tenant and client id (optional from hadoop-azure 3.4.1)") { + assume(handles.hasFixedSasTokenProvider, "hadoop-azure 3.3.4 makes both keys mandatory") + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.MSI, + acct(Keys.MSI_AUTHORITY) -> "") + outcome shouldBe Resolved( + "OAuth", + Some(Providers.MSI), + Map( + Keys.MSI_ENDPOINT -> "http://169.254.169.254/metadata/identity/oauth2/token", + Keys.MSI_AUTHORITY -> "https://login.microsoftonline.com/")) + } + + test("Resolved - Workload Identity with the default token file (hadoop-azure 3.4.1+)") { + assume(handles.hasFixedSasTokenProvider, "no WorkloadIdentityTokenProvider on 3.3.4") + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.WORKLOAD_IDENTITY, + acct(Keys.MSI_TENANT) -> "tenant-abc", + acct(Keys.CLIENT_ID) -> "client-123") + outcome shouldBe Resolved( + "OAuth", + Some(Providers.WORKLOAD_IDENTITY), + Map( + Keys.MSI_AUTHORITY -> "https://login.microsoftonline.com/", + Keys.MSI_TENANT -> "tenant-abc", + Keys.CLIENT_ID -> "client-123", + Keys.TOKEN_FILE -> "/var/run/secrets/azure/tokens/azure-identity-token")) + } + + test("Resolved - container-scoped keys win on 3.4.2+ and are ignored before") { + assume(handles.hasFixedSasTokenProvider, "no WorkloadIdentityTokenProvider on 3.3.4") + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "OAuth", + acct(Keys.OAUTH_PROVIDER_TYPE) -> Providers.WORKLOAD_IDENTITY, + acct(Keys.MSI_TENANT) -> "tenant-abc", + acct(Keys.CLIENT_ID) -> "account-client", + cont(Keys.CLIENT_ID) -> "container-client", + cont(Keys.TOKEN_FILE) -> " /mnt/tokens/container-token ") + val expectedClient = + if (handles.hasContainerScopedConfiguration) "container-client" else "account-client" + val expectedTokenFile = + if (handles.hasContainerScopedConfiguration) "/mnt/tokens/container-token" + else "/var/run/secrets/azure/tokens/azure-identity-token" + outcome shouldBe Resolved( + "OAuth", + Some(Providers.WORKLOAD_IDENTITY), + Map( + Keys.MSI_AUTHORITY -> "https://login.microsoftonline.com/", + Keys.MSI_TENANT -> "tenant-abc", + Keys.CLIENT_ID -> expectedClient, + Keys.TOKEN_FILE -> expectedTokenFile)) + } + + test("Resolved - fixed SAS token is trimmed and forwarded (hadoop-azure 3.4.1+)") { + assume(handles.hasFixedSasTokenProvider, "no fixed SAS token on 3.3.4") + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "SAS", + acct(Keys.SAS_FIXED_TOKEN) -> " ?sv=2020-08-04&sig=fixed-signature ") + outcome shouldBe + Resolved("SAS", None, Map(Keys.SAS_FIXED_TOKEN -> "?sv=2020-08-04&sig=fixed-signature")) + toOptions(outcome) should not contain key(Markers.SAS_PROVIDER_CLASS) + } + + test("Failed - SAS without a provider class on hadoop-azure 3.3.4, which has no fixed token") { + assume(!handles.hasFixedSasTokenProvider, "3.4.1+ accept the fixed token") + val outcome = resolveWith( + acct(Keys.AUTH_TYPE) -> "SAS", + acct(Keys.SAS_FIXED_TOKEN) -> "sv=2020-08-04&sig=fixed-signature") + outcome shouldBe a[Failed] + outcome.asInstanceOf[Failed].exceptionClass shouldBe tokenAccessProviderException + assertNoAzureKeys(toOptions(outcome)) + } + + test("Failed - SAS with neither class nor token (hadoop-azure 3.4.1+)") { + assume(handles.hasFixedSasTokenProvider, "3.3.4 fails differently, covered above") + val outcome = resolveWith(acct(Keys.AUTH_TYPE) -> "SAS", acct(Keys.SAS_FIXED_TOKEN) -> " ") + outcome shouldBe a[Failed] + outcome.asInstanceOf[Failed].exceptionClass shouldBe + "org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException" + } +} + +/** Records the key of every `getPassword` lookup, each of which may load a keystore. */ +class CountingConfiguration extends Configuration { + private val reads = ArrayBuffer[String]() + + def passwordReads: Seq[String] = reads.synchronized(reads.toList) + + override def getPassword(name: String): Array[Char] = { + reads.synchronized(reads += name) + super.getPassword(name) + } +} + +/** A delegating loader that refuses `AbfsConfiguration` until `blocked` is cleared. */ +class FlippingLoader(parent: ClassLoader) extends ClassLoader(parent) { + @volatile var blocked = true + + override def loadClass(name: String, resolve: Boolean): Class[_] = { + if (blocked && name == AbfsReflection.ClassNames.ABFS_CONFIGURATION) { + throw new ClassNotFoundException(name) + } + super.loadClass(name, resolve) + } +} + +object SecretLeakingKeyProvider { + val marker = "vault-secret-marker-7b3e9c" +} + +/** A KeyProvider whose failure quotes a secret the resolver never read from the conf. */ +class SecretLeakingKeyProvider extends KeyProvider { + override def getStorageAccountKey(accountName: String, conf: Configuration): String = + throw new IllegalArgumentException(s"bad key ${SecretLeakingKeyProvider.marker}") +} + +/** Fails loudly if anything constructs it: the resolver must only ever name this class. */ +class NeverInstantiatedSasTokenProvider extends SASTokenProvider { + NeverInstantiated.fail("NeverInstantiatedSasTokenProvider") + + override def initialize(configuration: Configuration, accountName: String): Unit = () + + override def getSASToken( + account: String, + fileSystem: String, + path: String, + operation: String): String = + throw new UnsupportedOperationException("not reached") +} + +/** An `AccessTokenProvider` that is not one of Hadoop's built-ins; never constructed. */ +class NeverInstantiatedAccessTokenProvider extends AccessTokenProvider { + NeverInstantiated.fail("NeverInstantiatedAccessTokenProvider") + + override protected def refreshToken(): AzureADToken = + throw new UnsupportedOperationException("not reached") +} + +/** Handles whose first call fails with a `LinkageError`, as a mismatched hadoop-azure would. */ +object LinkageStub extends Handles { + // Built here so the file has no `throw new ...Error(` (scalastyle); thrown below. + private val linkageFailure = new NoSuchMethodError("stub: AbfsConfiguration.") + + private def notReached: Nothing = throw new UnsupportedOperationException("not reached") + + override def hasContainerScopedConfiguration: Boolean = false + override def hasFixedSasTokenProvider: Boolean = false + override def hasClientAssertionProvider: Boolean = false + override def accessTokenProviderClass: Class[_] = classOf[AnyRef] + override def sasTokenProviderClass: Class[_] = classOf[AnyRef] + override def newAbfsConfiguration( + conf: Configuration, + account: String, + container: String, + uri: URI): AnyRef = throw linkageFailure + override def newKeyProviderConfiguration(conf: Configuration, account: String): AnyRef = + notReached + override def getAuthType(abfsConf: AnyRef, account: String): Enum[_] = notReached + override def getPasswordString(abfsConf: AnyRef, key: String): Option[String] = notReached + override def read( + abfsConf: AnyRef, + key: String, + password: Boolean): Either[Throwable, Option[String]] = notReached + override def getTokenProviderClass( + abfsConf: AnyRef, + authType: Enum[_], + key: String, + xface: Class[_]): Option[Class[_]] = notReached + override def getStorageAccountKey(abfsConf: AnyRef): String = notReached + override def getTokenProvider(abfsConf: AnyRef): Unit = notReached + override def getSASTokenProvider(abfsConf: AnyRef): Unit = notReached + override def authDefault(fieldName: String): String = notReached +} + +/** Throws from a constructor without leaving the compiler a dead-code warning to report. */ +object NeverInstantiated { + def fail(name: String): Unit = throw new IllegalStateException(s"$name was constructed") +} diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala new file mode 100644 index 00000000000..016491d8af8 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.objectstore + +import java.io.File +import java.nio.file.{Files, Paths} + +import org.apache.commons.io.FileUtils +import org.apache.spark.SparkConf +import org.apache.spark.sql.{CometTestBase, DataFrame} +import org.apache.spark.sql.comet.CometNativeScanExec +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{IntegerType, StructType} + +import org.apache.comet.{CometConf, ExtendedExplainInfo} +import org.apache.comet.hadoop.fs.FakeAbfsFileSystem + +/** + * Plans scans over `abfss://` paths served by [[FakeAbfsFileSystem]], with authentication + * resolved offline by the hadoop-azure on the test classpath. A scan Comet declines runs on + * Spark's own scan against the local files. + */ +class AbfsScanFallbackSuite extends CometTestBase { + + private val multipleAuthoritiesReason = "more than one container or account" + + private var rootDir: File = _ + + override protected def sparkConf: SparkConf = + super.sparkConf + .set("spark.hadoop.fs.abfss.impl", classOf[FakeAbfsFileSystem].getName) + .set("spark.hadoop.fs.abfss.impl.disable.cache", "true") + + override def beforeAll(): Unit = { + // java.io.tmpdir is under target/, which a fresh checkout may not have yet. + val tmp = Files.createDirectories(Paths.get(System.getProperty("java.io.tmpdir"))) + rootDir = Files.createTempDirectory(tmp, "comet-abfs-scan").toFile + super.beforeAll() + } + + protected override def afterAll(): Unit = { + if (rootDir != null) FileUtils.deleteDirectory(rootDir) + super.afterAll() + } + + private def localDir(name: String): String = new File(rootDir, name).getAbsolutePath + + private def abfss(container: String, account: String, dir: String): String = + s"abfss://$container@$account.dfs.core.windows.net${localDir(dir)}" + + private def accountKeys(keys: (String, String)*): Seq[(String, String)] = + keys.map { case (account, key) => + s"fs.azure.account.key.$account.dfs.core.windows.net" -> key + } + + private def writeParquet(dir: String, from: Int): Unit = + spark.range(from.toLong, from + 10L).toDF("id").write.parquet(localDir(dir)) + + private def writeCsv(dir: String, from: Int): Unit = + spark + .range(from.toLong, from + 10L) + .selectExpr("cast(id as int) as a", "cast(id * 2 as int) as b") + .write + .csv(localDir(dir)) + + private val csvConfs = + Seq(CometConf.COMET_CSV_V2_NATIVE_ENABLED.key -> "true", SQLConf.USE_V1_SOURCE_LIST.key -> "") + + private def readCsv(paths: String*): DataFrame = + spark.read + .schema(new StructType().add("a", IntegerType).add("b", IntegerType)) + .csv(paths: _*) + + test("native Parquet scan over two accounts with different keys falls back to Spark") { + writeParquet("parquet-a", 0) + writeParquet("parquet-b", 100) + withSQLConf(accountKeys("accta" -> "a2V5LWE=", "acctb" -> "a2V5LWI="): _*) { + val df = + spark.read.parquet( + abfss("data", "accta", "parquet-a"), + abfss("data", "acctb", "parquet-b")) + checkSparkAnswerAndFallbackReason(df, multipleAuthoritiesReason) + } + } + + test("native Parquet scan over two containers of one account falls back to Spark") { + writeParquet("parquet-c", 0) + writeParquet("parquet-d", 100) + withSQLConf(accountKeys("accta" -> "a2V5LWE="): _*) { + val df = + spark.read.parquet( + abfss("data", "accta", "parquet-c"), + abfss("logs", "accta", "parquet-d")) + checkSparkAnswerAndFallbackReason(df, multipleAuthoritiesReason) + } + } + + test("native Parquet scan over two directories of one container stays native") { + writeParquet("parquet-e", 0) + writeParquet("parquet-f", 100) + val confs = (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") +: + accountKeys("accta" -> "a2V5LWE=") + withSQLConf(confs: _*) { + // Plan only: running the native scan would send its requests to Azure. + val plan = spark.read + .parquet(abfss("data", "accta", "parquet-e"), abfss("data", "accta", "parquet-f")) + .queryExecution + .executedPlan + assert(plan.collect { case scan: CometNativeScanExec => scan }.size == 1, plan) + val reasons = new ExtendedExplainInfo().getFallbackReasons(plan) + assert(!reasons.exists(_.contains("ABFS")), reasons) + } + } + + test("native CSV scan over two accounts with different keys falls back to Spark") { + writeCsv("csv-a", 0) + writeCsv("csv-b", 100) + withSQLConf(csvConfs ++ accountKeys("accta" -> "a2V5LWE=", "acctb" -> "a2V5LWI="): _*) { + val df = readCsv(abfss("data", "accta", "csv-a"), abfss("data", "acctb", "csv-b")) + checkSparkAnswerAndFallbackReason(df, multipleAuthoritiesReason) + } + } + + test("native CSV scan over two containers of one account falls back to Spark") { + writeCsv("csv-c", 0) + writeCsv("csv-d", 100) + withSQLConf(csvConfs ++ accountKeys("accta" -> "a2V5LWE="): _*) { + val df = readCsv(abfss("data", "accta", "csv-c"), abfss("logs", "accta", "csv-d")) + checkSparkAnswerAndFallbackReason(df, multipleAuthoritiesReason) + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 1cfd1d36b92..d62733aa421 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -26,7 +26,7 @@ import org.scalatest.matchers.should.Matchers import org.apache.hadoop.conf.Configuration -import org.apache.comet.CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY +import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} class NativeConfigSuite extends AnyFunSuite with Matchers { @@ -81,38 +81,44 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { assert(unsupportedOptions.isEmpty, "Unsupported scheme should return empty options") } - test("extractObjectStoreOptions - ABFS forwards Hadoop fs.azure.* auth keys") { - // ABFS auth (account keys, OAuth, MSI/Workload Identity, SAS) lives under fs.azure.*, not - // fs.abfs.*. Verify abfs[s] forwards fs.azure.* (earlier versions dropped these credentials). + test("extractObjectStoreOptions - ABFS forwards the auth Hadoop resolves for the account") { + // ABFS auth lives under fs.azure.*; Hadoop (AbfsConfiguration) picks the mechanism and the + // values for the URI's account, and only those travel, under the global key names, with the + // comet.azure.* markers. Account-scoped keys and other accounts' keys stay on the driver. val hadoopConf = new Configuration() hadoopConf.set("fs.azure.account.auth.type.myacct.dfs.core.windows.net", "OAuth") hadoopConf.set( "fs.azure.account.oauth.provider.type.myacct.dfs.core.windows.net", - "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider") + "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider") + hadoopConf.set( + "fs.azure.account.oauth2.client.endpoint.myacct.dfs.core.windows.net", + "https://login.microsoftonline.com/tenant-abc/oauth2/token") hadoopConf.set("fs.azure.account.oauth2.client.id.myacct.dfs.core.windows.net", "client-123") - hadoopConf.set("fs.azure.account.oauth2.msi.tenant.myacct.dfs.core.windows.net", "tenant-abc") hadoopConf.set( - "fs.azure.account.oauth2.token.file.myacct.dfs.core.windows.net", - "/var/run/secrets/azure/tokens/azure-identity-token") + "fs.azure.account.oauth2.client.secret.myacct.dfs.core.windows.net", + "client-secret-value") + hadoopConf.set( + "fs.azure.account.oauth2.client.secret.other.dfs.core.windows.net", + "other-secret-value") Seq( "abfs://data@myacct.dfs.core.windows.net/path/file.parquet", "abfss://data@myacct.dfs.core.windows.net/path/file.parquet").foreach { path => val opts = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI(path)) - assert( - opts("fs.azure.account.oauth2.client.id.myacct.dfs.core.windows.net") == "client-123", - s"client id should be forwarded for $path") - assert( - opts("fs.azure.account.oauth2.msi.tenant.myacct.dfs.core.windows.net") == "tenant-abc", - s"tenant id should be forwarded for $path") - assert( - opts("fs.azure.account.oauth2.token.file.myacct.dfs.core.windows.net") == - "/var/run/secrets/azure/tokens/azure-identity-token", - s"federated token file should be forwarded for $path") - assert( - opts("fs.azure.account.oauth.provider.type.myacct.dfs.core.windows.net") == - "org.apache.hadoop.fs.azurebfs.oauth2.WorkloadIdentityTokenProvider", - s"oauth provider type should be forwarded for $path") + withClue(s"$path: $opts") { + assert(opts("comet.azure.resolution") == "resolved") + assert(opts("comet.azure.auth.type") == "OAuth") + assert( + opts("comet.azure.oauth.provider.class") == + "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider") + assert( + opts("fs.azure.account.oauth2.client.endpoint") == + "https://login.microsoftonline.com/tenant-abc/oauth2/token") + assert(opts("fs.azure.account.oauth2.client.id") == "client-123") + assert(opts("fs.azure.account.oauth2.client.secret") == "client-secret-value") + assert(!opts.keys.exists(_.endsWith(".myacct.dfs.core.windows.net"))) + assert(!opts.values.exists(_ == "other-secret-value")) + } } } @@ -354,4 +360,133 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { NativeConfig.resolveS3CompliantSchemes(conf) == Set("blob", "minio", "r2"), "schemes must be split on commas, trimmed, lowercased, with blanks dropped") } + + test("extractObjectStoreOptions - abfs without Azure auth keys is 'none' with no fs.* keys") { + // The one case where native may read the AZURE_* environment. A key set under the comet + // marker name in the Hadoop conf is never forwarded; the resolver's verdict replaces it. + val hadoopConf = new Configuration() + hadoopConf.set(COMET_LIBHDFS_SCHEMES_KEY, "hdfs") + hadoopConf.set("comet.azure.resolution", "resolved") + hadoopConf.set("fs.azure.account.hns.enabled", "true") + + Seq("abfs", "abfss").foreach { scheme => + val opts = NativeConfig.extractObjectStoreOptions( + hadoopConf, + new URI(s"$scheme://data@myacct.dfs.core.windows.net/path/file.parquet")) + withClue(s"$scheme: $opts") { + assert( + opts == Map(COMET_LIBHDFS_SCHEMES_KEY -> "hdfs", "comet.azure.resolution" -> "none")) + } + } + } + + test("extractObjectStoreOptions - abfs resolution failures travel as error markers only") { + val hadoopConf = new Configuration() + hadoopConf.set("fs.azure.account.key.myacct.dfs.core.windows.net", "not-base64-at-all!!!") + + val opts = NativeConfig.extractObjectStoreOptions( + hadoopConf, + new URI("abfss://data@myacct.dfs.core.windows.net/path/file.parquet")) + assert(opts("comet.azure.resolution") == "error", opts.toString) + assert(opts("comet.azure.error.class").endsWith("KeyProviderException"), opts.toString) + assert(!opts.keys.exists(_.startsWith("fs.")), opts.toString) + assert(!opts.values.exists(_.contains("not-base64-at-all!!!")), opts.toString) + } + + test("extractObjectStoreOptions - wasb still forwards fs.azure.* by prefix, without markers") { + val hadoopConf = new Configuration() + hadoopConf.set("fs.azure.account.key.myacct.blob.core.windows.net", "azure-key") + hadoopConf.set("fs.azure.sas.fixed.token", "sv=2020-08-04&sig=fixed") + + Seq( + "wasb://data@myacct.blob.core.windows.net/key", + "wasbs://data@myacct.blob.core.windows.net/key").foreach { path => + val opts = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI(path)) + withClue(s"$path: $opts") { + assert(opts("fs.azure.account.key.myacct.blob.core.windows.net") == "azure-key") + assert(opts("fs.azure.sas.fixed.token") == "sv=2020-08-04&sig=fixed") + assert(!opts.keys.exists(_.startsWith("comet.azure."))) + } + } + Seq("s3a://bucket/key", "gs://bucket/key").foreach { path => + val opts = NativeConfig.extractObjectStoreOptions(hadoopConf, new URI(path)) + assert(!opts.keys.exists(_.startsWith("comet.azure.")), s"$path: $opts") + } + } + + private def abfsFile(container: String, account: String, file: String): URI = + new URI(s"abfss://$container@$account.dfs.core.windows.net/path/$file") + + private def assertDeclined(hadoopConf: Configuration, uris: Seq[URI]): Unit = + NativeConfig.extractScanObjectStoreOptions(hadoopConf, uris) match { + case Left(reason) => + reason should include(uris.head.getRawAuthority) + reason should include(uris.last.getRawAuthority) + case Right(options) => fail(s"expected a decline, got $options") + } + + test("extractScanObjectStoreOptions - abfs accounts with different keys decline the scan") { + val hadoopConf = new Configuration() + hadoopConf.set("fs.azure.account.key.accta.dfs.core.windows.net", "a2V5LWE=") + hadoopConf.set("fs.azure.account.key.acctb.dfs.core.windows.net", "a2V5LWI=") + assertDeclined( + hadoopConf, + Seq( + abfsFile("data", "accta", "a1.parquet"), + abfsFile("data", "accta", "a2.parquet"), + abfsFile("data", "acctb", "b1.parquet"))) + } + + test("extractScanObjectStoreOptions - containers of one account decline the scan") { + // Native reads a whole partition through the store of its first file's container. + val hadoopConf = new Configuration() + hadoopConf.set("fs.azure.account.key.accta.dfs.core.windows.net", "a2V5LWE=") + assertDeclined( + hadoopConf, + Seq(abfsFile("data", "accta", "a1.parquet"), abfsFile("logs", "accta", "a2.parquet"))) + } + + test("extractScanObjectStoreOptions - accounts sharing a global key decline the scan") { + val hadoopConf = new Configuration() + hadoopConf.set("fs.azure.account.key", "Z2xvYmFs") + assertDeclined( + hadoopConf, + Seq(abfsFile("data", "accta", "a.parquet"), abfsFile("data", "acctb", "b.parquet"))) + } + + test("extractScanObjectStoreOptions - files in one container keep the first file's options") { + val hadoopConf = new Configuration() + hadoopConf.set("fs.azure.account.key.accta.dfs.core.windows.net", "a2V5LWE=") + val uris = (1 to 5).map(i => abfsFile("data", "accta", s"a$i.parquet")) + + val single = NativeConfig.extractObjectStoreOptions(hadoopConf, uris.head) + assert(single("fs.azure.account.key") == "a2V5LWE=", single.toString) + NativeConfig.extractScanObjectStoreOptions(hadoopConf, uris) shouldBe Right(single) + } + + test("extractScanObjectStoreOptions - abfs and non-abfs files in one scan decline it") { + val hadoopConf = new Configuration() + hadoopConf.set("fs.azure.account.key", "Z2xvYmFs") + val abfs = abfsFile("data", "accta", "a.parquet") + val s3 = new URI("s3a://bucket/path/b.parquet") + + Seq(Seq(abfs, s3), Seq(s3, abfs)).foreach { uris => + val result = NativeConfig.extractScanObjectStoreOptions(hadoopConf, uris) + assert(result.isLeft, s"$uris: $result") + } + } + + test("extractScanObjectStoreOptions - non-abfs scans keep the first file's options") { + val hadoopConf = new Configuration() + hadoopConf.set("fs.s3a.access.key", "s3-access-key") + hadoopConf.set("fs.s3a.bucket.other.access.key", "other-access-key") + val uris = Seq( + new URI("s3a://bucket/path/a.parquet"), + new URI("s3a://other/path/b.parquet"), + new URI("s3://bucket/path/c.parquet")) + + NativeConfig.extractScanObjectStoreOptions(hadoopConf, uris) shouldBe + Right(NativeConfig.extractObjectStoreOptions(hadoopConf, uris.head)) + NativeConfig.extractScanObjectStoreOptions(hadoopConf, Seq.empty) shouldBe Right(Map.empty) + } }