Skip to content

output_azure: OAuth authentication support for Azure output plugins #11571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zshuang0316 wants to merge 4 commits into
base: master
Choose a base branch
from
Open

output_azure: OAuth authentication support for Azure output plugins #11571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
555bd8c
azure_auth: add common Azure authentication module
zshuang0316 Mar 17, 2026
e665cd4
out_azure_blob: add OAuth authentication support
zshuang0316 Mar 17, 2026
091d869
out_azure_kusto: migrate to common Azure authentication
zshuang0316 Mar 17, 2026
710e330
tests: add unit tests for Azure Blob OAuth authentication
zshuang0316 Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 107 additions & 0 deletions include/fluent-bit/flb_azure_auth.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/* Fluent Bit
* ==========
* Copyright (C) 2015-2026 The Fluent Bit Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef FLB_AZURE_AUTH_H
#define FLB_AZURE_AUTH_H

#include <fluent-bit/flb_info.h>
#include <fluent-bit/flb_oauth2.h>
#include <fluent-bit/flb_sds.h>

/* Authentication types for Azure services */
typedef enum {
FLB_AZURE_AUTH_KEY = 0, /* Shared Access Key (blob-specific) */
FLB_AZURE_AUTH_SAS, /* Shared Access Signature (blob-specific) */
FLB_AZURE_AUTH_SERVICE_PRINCIPAL, /* Service Principal (Client ID + Secret) */
FLB_AZURE_AUTH_MANAGED_IDENTITY_SYSTEM, /* System-assigned Managed Identity */
FLB_AZURE_AUTH_MANAGED_IDENTITY_USER, /* User-assigned Managed Identity */
FLB_AZURE_AUTH_WORKLOAD_IDENTITY /* Workload Identity (Federated Token) */
} flb_azure_auth_type;

/* Azure Instance Metadata Service (IMDS) endpoint for Managed Identity */
#define FLB_AZURE_IMDS_HOST "169.254.169.254"
#define FLB_AZURE_IMDS_PORT "80"

/* Managed Identity authentication URL template */
#define FLB_AZURE_MSI_AUTH_URL_TEMPLATE \
"http://169.254.169.254/metadata/identity/oauth2/token?api-version=2021-02-01%s%s&resource=%s"

/* Microsoft Authentication Library (MSAL) authorization URL template */
#define FLB_AZURE_MSAL_AUTH_URL_TEMPLATE \
"https://login.microsoftonline.com/%s/oauth2/v2.0/token"

/* Azure Blob Storage resource identifier */
#define FLB_AZURE_BLOB_RESOURCE "https://storage.azure.com"

/* Azure Kusto resource identifier */
#define FLB_AZURE_KUSTO_RESOURCE "https://help.kusto.windows.net"

/* Default Workload Identity token file path */
#define FLB_AZURE_WORKLOAD_IDENTITY_TOKEN_FILE \
"/var/run/secrets/azure/tokens/azure-identity-token"

/**
* Get an OAuth2 access token using Azure Managed Identity (MSI)
*
* This function retrieves an access token from the Azure Instance Metadata Service (IMDS)
* for use with Azure services. Supports both system-assigned and user-assigned managed identities.
*
* @param ctx OAuth2 context containing connection and token information
* @return Access token string on success, NULL on failure
*/
char *flb_azure_msi_token_get(struct flb_oauth2 *ctx);

/**
* Get an OAuth2 access token using Azure Workload Identity
*
* This function exchanges a federated token (JWT) for an Azure AD access token
* using the OAuth2 client credentials flow with client assertion.
*
* @param ctx OAuth2 context for token management
* @param token_file Path to the file containing the federated token
* @param client_id Client ID of the Azure AD application
* @param tenant_id Tenant ID of the Azure AD directory
* @param resource Resource scope for the token (e.g., "https://storage.azure.com/")
* @return 0 on success, -1 on failure
*/
int flb_azure_workload_identity_token_get(struct flb_oauth2 *ctx,
const char *token_file,
const char *client_id,
const char *tenant_id,
const char *resource);

/**
* Build OAuth URL for Azure authentication
*
* Creates the appropriate OAuth2 endpoint URL based on the authentication type.
* For Managed Identity, uses IMDS endpoint. For Service Principal and Workload Identity,
* uses Azure AD OAuth2 endpoint.
*
* @param auth_type Type of authentication to use
* @param tenant_id Azure AD tenant ID (required for Service Principal and Workload Identity)
* @param client_id Client ID (optional, used for user-assigned managed identity)
* @param resource Resource scope for the token (e.g., storage.azure.com)
* @return Allocated SDS string with OAuth URL, or NULL on failure
*/
flb_sds_t flb_azure_auth_build_oauth_url(flb_azure_auth_type auth_type,
const char *tenant_id,
const char *client_id,
const char *resource);

#endif /* FLB_AZURE_AUTH_H */
27 changes: 26 additions & 1 deletion plugins/out_azure_blob/azure_blob.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1826,7 +1826,7 @@ static struct flb_config_map config_map[] = {
{
FLB_CONFIG_MAP_STR, "auth_type", "key",
0, FLB_TRUE, offsetof(struct flb_azure_blob, auth_type),
"Set the auth type: key or sas"
"Set the auth type: key, sas, service_principal, managed_identity, or workload_identity"
},

{
Expand All @@ -1835,6 +1835,31 @@ static struct flb_config_map config_map[] = {
"Azure Blob SAS token"
},

/* OAuth authentication parameters */
{
FLB_CONFIG_MAP_STR, "tenant_id", NULL,
0, FLB_TRUE, offsetof(struct flb_azure_blob, tenant_id),
"Azure AD tenant ID (required for service_principal and workload_identity auth)"
},

{
FLB_CONFIG_MAP_STR, "client_id", NULL,
0, FLB_TRUE, offsetof(struct flb_azure_blob, client_id),
"Azure AD client ID / Application ID (required for OAuth-based authentication)"
},

{
FLB_CONFIG_MAP_STR, "client_secret", NULL,
0, FLB_TRUE, offsetof(struct flb_azure_blob, client_secret),
"Azure AD client secret (required for service_principal auth)"
},

{
FLB_CONFIG_MAP_STR, "workload_identity_token_file", NULL,
0, FLB_TRUE, offsetof(struct flb_azure_blob, workload_identity_token_file),
"Path to workload identity token file (for workload_identity auth)"
},

{
FLB_CONFIG_MAP_STR, "database_file", NULL,
0, FLB_TRUE, offsetof(struct flb_azure_blob, database_file),
Expand Down
23 changes: 19 additions & 4 deletions plugins/out_azure_blob/azure_blob.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
#include <fluent-bit/flb_upstream.h>
#include <fluent-bit/flb_sds.h>
#include <fluent-bit/flb_sqldb.h>
#include <fluent-bit/flb_azure_auth.h>
#ifdef FLB_HAVE_TLS
#include <fluent-bit/flb_oauth2.h>
#endif

/* Content-Type */
#define AZURE_BLOB_CT "Content-Type"
Expand All @@ -48,9 +52,6 @@
#define AZURE_BLOB_APPENDBLOB 0
#define AZURE_BLOB_BLOCKBLOB 1

#define AZURE_BLOB_AUTH_KEY 0
#define AZURE_BLOB_AUTH_SAS 1

struct flb_azure_blob {
int auto_create_container;
int emulator_mode;
Expand All @@ -65,6 +66,13 @@ struct flb_azure_blob {
flb_sds_t date_key;
flb_sds_t auth_type;
flb_sds_t sas_token;

/* OAuth authentication fields */
flb_sds_t tenant_id;
flb_sds_t client_id;
flb_sds_t client_secret;
flb_sds_t workload_identity_token_file;

flb_sds_t database_file;
size_t part_size;
time_t upload_parts_timeout;
Expand Down Expand Up @@ -112,7 +120,7 @@ struct flb_azure_blob {
* Internal use
*/
int btype; /* blob type */
int atype; /* auth type */
flb_azure_auth_type atype; /* auth type (uses flb_azure_auth_type enum from flb_azure_auth.h) */
flb_sds_t real_endpoint;
flb_sds_t base_uri;
flb_sds_t shared_key_prefix;
Expand All @@ -121,6 +129,13 @@ struct flb_azure_blob {
unsigned char *decoded_sk; /* decoded shared key */
size_t decoded_sk_size; /* size of decoded shared key */

#ifdef FLB_HAVE_TLS
/* OAuth2 authentication */
flb_sds_t oauth_url;
struct flb_oauth2 *o;
pthread_mutex_t token_mutex;
Comment on lines +133 to +136
Copy link

@coderabbitai coderabbitai bot Mar 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Check mutex initialization and destruction for azure_blob
echo "=== Checking token_mutex initialization ==="
rg -n "pthread_mutex_init.*token_mutex|token_mutex.*pthread_mutex_init" plugins/out_azure_blob/

echo ""
echo "=== Checking token_mutex destruction ==="
rg -n "pthread_mutex_destroy.*token_mutex|token_mutex.*pthread_mutex_destroy" plugins/out_azure_blob/

echo ""
echo "=== Check azure_blob_conf.c for mutex handling ==="
rg -n -B2 -A2 "token_mutex" plugins/out_azure_blob/azure_blob_conf.c

Repository: fluent/fluent-bit

Length of output: 779

🏁 Script executed:

# Check the function context for mutex init and destroy operations
echo "=== Context around mutex initialization (line 800) ==="
sed -n '770,810p' plugins/out_azure_blob/azure_blob_conf.c

echo ""
echo "=== Context around mutex destruction (line 941) ==="
sed -n '920,960p' plugins/out_azure_blob/azure_blob_conf.c

echo ""
echo "=== Overall structure of azure_blob_conf.c to understand lifecycle ==="
rg -n "^[a-zA-Z_].*\(.*\).*\{" plugins/out_azure_blob/azure_blob_conf.c | head -20

Repository: fluent/fluent-bit

Length of output: 2497

🏁 Script executed:

# Get function names and full context
echo "=== Function containing initialization (around line 800) ==="
sed -n '650,810p' plugins/out_azure_blob/azure_blob_conf.c | grep -B 150 "Initialize token mutex" | head -20

echo ""
echo "=== Function containing destruction (around line 941) ==="
sed -n '850,960p' plugins/out_azure_blob/azure_blob_conf.c | grep -B 100 "pthread_mutex_destroy" | head -20

echo ""
echo "=== Check mutex usage in the codebase ==="
rg -n "token_mutex" plugins/out_azure_blob/ -B1 -A1

Repository: fluent/fluent-bit

Length of output: 3949

🏁 Script executed:

# Get the function signatures
echo "=== Function containing initialization ==="
awk '/^[a-zA-Z_].*\{/,/pthread_mutex_init.*token_mutex/ {print NR": "$0}' plugins/out_azure_blob/azure_blob_conf.c | tail -100 | head -50

echo ""
echo "=== Function containing destruction ==="
awk '/^[a-zA-Z_].*\{/,/pthread_mutex_destroy.*token_mutex/ {print NR": "$0}' plugins/out_azure_blob/azure_blob_conf.c | tail -100 | head -50

echo ""
echo "=== Check error handling for pthread_mutex_init ==="
rg -n -B5 -A5 "pthread_mutex_init.*token_mutex" plugins/out_azure_blob/

echo ""
echo "=== Verify these are plugin init and exit functions ==="
grep -n "^static.*flb_output_plugin\|^static.*plugin_exit\|^static.*plugin_init\|^static.*config_destroy\|^flb_output_plugin_" plugins/out_azure_blob/azure_blob_conf.c | head -20

Repository: fluent/fluent-bit

Length of output: 1055

🏁 Script executed:

# Try to find the function definitions more directly
echo "=== Find function declarations in azure_blob_conf.c ==="
rg -n "^[a-zA-Z_][a-zA-Z0-9_*\s]+\(" plugins/out_azure_blob/azure_blob_conf.c | head -30

echo ""
echo "=== Search for specific function patterns ==="
grep -n "^static.*\*" plugins/out_azure_blob/azure_blob_conf.c | head -20

echo ""
echo "=== Look for function at line 800 context ==="
sed -n '700,820p' plugins/out_azure_blob/azure_blob_conf.c | head -50

echo ""
echo "=== Look for function at line 941 context ==="
sed -n '850,960p' plugins/out_azure_blob/azure_blob_conf.c | tail -100

Repository: fluent/fluent-bit

Length of output: 5595

Add error handling for mutex initialization.

The token_mutex is properly initialized in flb_azure_blob_conf_create() and destroyed in flb_azure_blob_conf_destroy(), preventing resource leaks. However, pthread_mutex_init() at line 800 does not check the return value; if initialization fails, the error is silently ignored and subsequent mutex operations may fail unexpectedly.

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@plugins/out_azure_blob/azure_blob.h` around lines 134 - 137,
pthread_mutex_init() return value is not checked in
flb_azure_blob_conf_create(), so if mutex initialization for token_mutex fails
later pthread mutex operations will crash; update flb_azure_blob_conf_create()
to check the result of pthread_mutex_init(&conf->token_mutex, NULL), handle
non-zero return by logging an error (use existing logging facility), clean up
allocated resources and return NULL (or an error code) so caller knows creation
failed, and ensure flb_azure_blob_conf_destroy() still safely handles
partially-initialized conf structures by only calling
pthread_mutex_destroy(&conf->token_mutex) when initialization succeeded.

#endif

#ifdef FLB_HAVE_SQLDB
/*
* SQLite by default is not built with multi-threading enabled, and
Expand Down
2 changes: 1 addition & 1 deletion plugins/out_azure_blob/azure_blob_appendblob.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag)
flb_sds_printf(&uri, "/%s?comp=appendblock", tag);
}

if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) {
if (ctx->atype == FLB_AZURE_AUTH_SAS && ctx->sas_token) {
flb_sds_printf(&uri, "&%s", ctx->sas_token);
}

Expand Down
6 changes: 3 additions & 3 deletions plugins/out_azure_blob/azure_blob_blockblob.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name)
flb_sds_printf(&uri, "/%s?comp=blocklist", name);
}

if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) {
if (ctx->atype == FLB_AZURE_AUTH_SAS && ctx->sas_token) {
flb_sds_printf(&uri, "&%s", ctx->sas_token);
}

Expand Down Expand Up @@ -103,7 +103,7 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
}
}

if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) {
if (ctx->atype == FLB_AZURE_AUTH_SAS && ctx->sas_token) {
flb_sds_printf(&uri, "&%s", ctx->sas_token);
}

Expand Down Expand Up @@ -137,7 +137,7 @@ flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
flb_sds_printf(&uri, "/%s.%s.%" PRIu64 "%s?comp=blocklist", tag, str, ms, ext);
}

if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) {
if (ctx->atype == FLB_AZURE_AUTH_SAS && ctx->sas_token) {
flb_sds_printf(&uri, "&%s", ctx->sas_token);
}

Expand Down
Loading
Loading