From 7110977acc519d1e8aa5e84972964d3542e42ead Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Thu, 5 Feb 2026 15:39:20 -0800 Subject: [PATCH 01/15] Add SqlBulkCopyOptions.CacheMetadata flag --- .../Microsoft.Data.SqlClient/SqlBulkCopy.xml | 13 + .../SqlBulkCopyOptions.xml | 13 + .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 44 +++ .../Data/SqlClient/SqlBulkCopyOptions.cs | 3 + .../SqlBulkCopyCacheMetadataTest.cs | 97 ++++++ ....Data.SqlClient.ManualTesting.Tests.csproj | 3 +- .../SQL/SqlBulkCopyTest/CacheMetadata.cs | 319 ++++++++++++++++++ .../SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs | 36 ++ 8 files changed, 527 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml index 461ece2b2a..bf6e3c014b 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml @@ -235,6 +235,19 @@ This code is provided to demonstrate the syntax for using **SqlBulkCopy** only. ]]> + + + Clears the cached destination table metadata when using the option. + + + + Call this method when you know the destination table schema has changed and you want to force the next operation to refresh the metadata from the server. + + + The cache is automatically invalidated when the property is changed to a different table name. + + + Enables or disables a object to stream data from an object diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml index a4ac472666..79c3ef5c3a 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml @@ -68,5 +68,18 @@ To see how the option changes the way the bulk load works, run the sample with t When specified, each batch of the bulk-copy operation will occur within a transaction. If you indicate this option and also provide a object to the constructor, an occurs. + + + + When specified, CacheMetadata caches destination table metadata after the first bulk copy operation, allowing subsequent operations to the same table to skip the metadata discovery query. This can improve performance when performing multiple bulk copy operations to the same destination table. + + + Warning: Use this option only when you are certain the destination table schema will not change between bulk copy operations. If the table schema changes (columns added, removed, or modified), using cached metadata may result in data corruption, failed operations, or unexpected behavior. Call to clear the cache if the schema changes. + + + The cache is automatically invalidated when is changed to a different table. + + + diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 39e4f570c7..d26a35fdbb 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -235,6 +235,10 @@ private int RowNumber private SourceColumnMetadata[] _currentRowMetadata; + // Metadata caching fields for CacheMetadata option + private BulkCopySimpleResultSet _cachedMetadata; + private string _cachedDestinationTableName; + #if DEBUG internal static bool s_setAlwaysTaskOnWrite; //when set and in DEBUG mode, TdsParser::WriteBulkCopyValue will always return a task internal static bool SetAlwaysTaskOnWrite @@ -353,6 +357,14 @@ public string DestinationTableName { throw ADP.ArgumentOutOfRange(nameof(DestinationTableName)); } + + // Invalidate cached metadata if the destination table name changes + if (!string.Equals(_destinationTableName, value, StringComparison.Ordinal)) + { + _cachedMetadata = null; + _cachedDestinationTableName = null; + } + _destinationTableName = value; } } @@ -497,6 +509,16 @@ IF EXISTS (SELECT TOP 1 * FROM sys.all_columns WHERE [object_id] = OBJECT_ID('sy // We need to have a _parser.RunAsync to make it real async. private Task CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result) { + // Check if we have valid cached metadata for the current destination table + if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata) && + _cachedMetadata != null && + string.Equals(_cachedDestinationTableName, _destinationTableName, StringComparison.Ordinal)) + { + SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName); + result = _cachedMetadata; + return null; + } + string TDSCommand = CreateInitialQuery(); SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Initial Query: '{0}'", TDSCommand); SqlClientEventSource.Log.TryCorrelationTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Correlation | Object Id {0}, Activity Id {1}", ObjectID, ActivityCorrelator.Current); @@ -506,6 +528,7 @@ private Task CreateAndExecuteInitialQueryAsync(out Bulk { result = new BulkCopySimpleResultSet(); RunParser(result); + CacheMetadataIfEnabled(result); return null; } else @@ -523,12 +546,23 @@ private Task CreateAndExecuteInitialQueryAsync(out Bulk { var internalResult = new BulkCopySimpleResultSet(); RunParserReliably(internalResult); + CacheMetadataIfEnabled(internalResult); return internalResult; } }, TaskScheduler.Default); } } + private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result) + { + if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata)) + { + _cachedMetadata = result; + _cachedDestinationTableName = _destinationTableName; + SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CacheMetadataIfEnabled | Info | Cached metadata for table '{0}'", _destinationTableName); + } + } + // Matches associated columns with metadata from initial query. // Builds and executes the update bulk command. private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults) @@ -880,6 +914,14 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults) _parser.WriteBulkCopyMetaData(metadataCollection, _sortedColumnMappings.Count, _stateObj); } + /// + public void InvalidateMetadataCache() + { + _cachedMetadata = null; + _cachedDestinationTableName = null; + SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.InvalidateMetadataCache | Info | Metadata cache invalidated"); + } + // Terminates the bulk copy operation. // Must be called at the end of the bulk copy session. /// @@ -900,6 +942,8 @@ private void Dispose(bool disposing) // Dispose dependent objects _columnMappings = null; _parser = null; + _cachedMetadata = null; + _cachedDestinationTableName = null; try { // Just in case there is a lingering transaction (which there shouldn't be) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopyOptions.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopyOptions.cs index 5454e609aa..5adbb21101 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopyOptions.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopyOptions.cs @@ -33,6 +33,9 @@ public enum SqlBulkCopyOptions /// AllowEncryptedValueModifications = 1 << 6, + + /// + CacheMetadata = 1 << 7, } } diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs new file mode 100644 index 0000000000..8d20c47d64 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Xunit; + +namespace Microsoft.Data.SqlClient.Tests +{ + public class SqlBulkCopyCacheMetadataTest + { + [Fact] + public void CacheMetadata_FlagValue_IsCorrect() + { + Assert.Equal(1 << 7, (int)SqlBulkCopyOptions.CacheMetadata); + } + + [Fact] + public void CacheMetadata_CanBeCombinedWithOtherOptions() + { + SqlBulkCopyOptions combined = + SqlBulkCopyOptions.CacheMetadata | + SqlBulkCopyOptions.KeepIdentity | + SqlBulkCopyOptions.TableLock; + + Assert.True((combined & SqlBulkCopyOptions.CacheMetadata) == SqlBulkCopyOptions.CacheMetadata); + Assert.True((combined & SqlBulkCopyOptions.KeepIdentity) == SqlBulkCopyOptions.KeepIdentity); + Assert.True((combined & SqlBulkCopyOptions.TableLock) == SqlBulkCopyOptions.TableLock); + } + + [Fact] + public void CacheMetadata_DoesNotOverlapExistingFlags() + { + int cacheMetadataValue = (int)SqlBulkCopyOptions.CacheMetadata; + Assert.NotEqual((int)SqlBulkCopyOptions.Default, cacheMetadataValue); + Assert.NotEqual((int)SqlBulkCopyOptions.KeepIdentity, cacheMetadataValue); + Assert.NotEqual((int)SqlBulkCopyOptions.CheckConstraints, cacheMetadataValue); + Assert.NotEqual((int)SqlBulkCopyOptions.TableLock, cacheMetadataValue); + Assert.NotEqual((int)SqlBulkCopyOptions.KeepNulls, cacheMetadataValue); + Assert.NotEqual((int)SqlBulkCopyOptions.FireTriggers, cacheMetadataValue); + Assert.NotEqual((int)SqlBulkCopyOptions.UseInternalTransaction, cacheMetadataValue); + Assert.NotEqual((int)SqlBulkCopyOptions.AllowEncryptedValueModifications, cacheMetadataValue); + } + + [Fact] + public void InvalidateMetadataCache_CanBeCalledWithoutError() + { + using SqlBulkCopy bulkCopy = new(new SqlConnection()); + bulkCopy.InvalidateMetadataCache(); + } + + [Fact] + public void InvalidateMetadataCache_CanBeCalledMultipleTimes() + { + using SqlBulkCopy bulkCopy = new(new SqlConnection()); + bulkCopy.InvalidateMetadataCache(); + bulkCopy.InvalidateMetadataCache(); + bulkCopy.InvalidateMetadataCache(); + } + + [Fact] + public void InvalidateMetadataCache_WithCacheMetadataOption() + { + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); + bulkCopy.InvalidateMetadataCache(); + } + + [Fact] + public void InvalidateMetadataCache_WithoutCacheMetadataOption() + { + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.Default, null); + bulkCopy.InvalidateMetadataCache(); + } + + [Fact] + public void DestinationTableName_Change_DoesNotThrowWithCacheMetadata() + { + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); + bulkCopy.DestinationTableName = "Table1"; + bulkCopy.DestinationTableName = "Table2"; + bulkCopy.DestinationTableName = "Table1"; + } + + [Fact] + public void Constructor_WithCacheMetadataOption_Succeeds() + { + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); + Assert.NotNull(bulkCopy); + } + + [Fact] + public void Constructor_WithCacheMetadataAndConnectionString_Succeeds() + { + using SqlBulkCopy bulkCopy = new("Server=localhost", SqlBulkCopyOptions.CacheMetadata); + Assert.NotNull(bulkCopy); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj index 01133f2ec2..53f0767000 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj @@ -111,6 +111,7 @@ + @@ -310,7 +311,7 @@ Utf8String - + diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs new file mode 100644 index 0000000000..551a9512d9 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs @@ -0,0 +1,319 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests +{ + public class CacheMetadata + { + private static readonly string sourceTable = "employees"; + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; + + // Test that CacheMetadata option works for multiple WriteToServer calls to the same table. + public static void Test(string srcConstr, string dstConstr, string dstTable) + { + string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable; + + // First WriteToServer: metadata is queried and cached. + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 5); + + // Second WriteToServer: should reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 10); + + // Third WriteToServer: should still reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 15); + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } + + public class CacheMetadataInvalidate + { + private static readonly string sourceTable = "employees"; + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; + + // Test that InvalidateMetadataCache forces a fresh metadata query. + public static void Test(string srcConstr, string dstConstr, string dstTable) + { + string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable; + + // First WriteToServer: metadata is queried and cached. + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 5); + + // Invalidate the cache and write again: should still succeed after re-querying metadata. + bulkcopy.InvalidateMetadataCache(); + + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 10); + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } + + public class CacheMetadataDestinationChange + { + private static readonly string sourceTable = "employees"; + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; + + // Test that changing DestinationTableName invalidates the cache and works correctly with a new table. + public static void Test(string srcConstr, string dstConstr, string dstTable1, string dstTable2) + { + string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); + string initialQuery1 = string.Format(initialQueryTemplate, dstTable1); + string initialQuery2 = string.Format(initialQueryTemplate, dstTable2); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery1); + Helpers.TryExecute(dstCmd, initialQuery2); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + + // Write to first table. + bulkcopy.DestinationTableName = dstTable1; + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable1, 3, 5); + + // Change destination table: cache should be invalidated automatically. + bulkcopy.DestinationTableName = dstTable2; + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable2, 3, 5); + } + finally + { + Helpers.TryDropTable(dstConstr, dstTable1); + Helpers.TryDropTable(dstConstr, dstTable2); + } + } + } + + public class CacheMetadataWithoutFlag + { + private static readonly string sourceTable = "employees"; + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; + + // Test that without the CacheMetadata flag, multiple writes still work (no regression). + public static void Test(string srcConstr, string dstConstr, string dstTable) + { + string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn); + bulkcopy.DestinationTableName = dstTable; + + // First WriteToServer without CacheMetadata. + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 5); + + // Second WriteToServer without CacheMetadata. + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 10); + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } + + public class CacheMetadataWithDataTable + { + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50), col3 nvarchar(50))"; + + // Test that CacheMetadata works with DataTable source as well as IDataReader. + public static void Test(string dstConstr, string dstTable) + { + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + DataTable sourceData = new(); + sourceData.Columns.Add("col1", typeof(int)); + sourceData.Columns.Add("col2", typeof(string)); + sourceData.Columns.Add("col3", typeof(string)); + sourceData.Rows.Add(1, "Alice", "Smith"); + sourceData.Rows.Add(2, "Bob", "Jones"); + sourceData.Rows.Add(3, "Charlie", "Brown"); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable; + + // First WriteToServer with DataTable: metadata is queried and cached. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 3); + + // Second WriteToServer with DataTable: should reuse cached metadata. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 6); + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } + + public class CacheMetadataCombinedWithKeepNulls + { + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50) default 'DefaultVal', col3 nvarchar(50))"; + + // Test that CacheMetadata works correctly when combined with other SqlBulkCopyOptions. + public static void Test(string dstConstr, string dstTable) + { + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + DataTable sourceData = new(); + sourceData.Columns.Add("col1", typeof(int)); + sourceData.Columns.Add("col2", typeof(string)); + sourceData.Columns.Add("col3", typeof(string)); + sourceData.Rows.Add(1, DBNull.Value, "Smith"); + sourceData.Rows.Add(2, "Bob", DBNull.Value); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata | SqlBulkCopyOptions.KeepNulls, null); + bulkcopy.DestinationTableName = dstTable; + bulkcopy.ColumnMappings.Add("col1", "col1"); + bulkcopy.ColumnMappings.Add("col2", "col2"); + bulkcopy.ColumnMappings.Add("col3", "col3"); + + // First write with CacheMetadata | KeepNulls. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 2); + + // Verify nulls were kept (not replaced by default values). + using SqlCommand verifyCmd = new("select col2 from " + dstTable + " where col1 = 1", dstConn); + object result = verifyCmd.ExecuteScalar(); + Assert.Equal(System.DBNull.Value, result); + + // Second write should reuse cached metadata. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 4); + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs index 4672adb242..21c24477a3 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs @@ -308,5 +308,41 @@ public void OrderHintIdentityColumnTest() { OrderHintIdentityColumn.Test(_connStr, AddGuid("SqlBulkCopyTest_OrderHintIdentityColumn")); } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataTest() + { + CacheMetadata.Test(_connStr, _connStr, AddGuid("SqlBulkCopyTest_CacheMetadata")); + } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataInvalidateTest() + { + CacheMetadataInvalidate.Test(_connStr, _connStr, AddGuid("SqlBulkCopyTest_CacheMetadataInvalidate")); + } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataDestinationChangeTest() + { + CacheMetadataDestinationChange.Test(_connStr, _connStr, AddGuid("SqlBulkCopyTest_CacheMetadataDstChange0"), AddGuid("SqlBulkCopyTest_CacheMetadataDstChange1")); + } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataWithoutFlagTest() + { + CacheMetadataWithoutFlag.Test(_connStr, _connStr, AddGuid("SqlBulkCopyTest_CacheMetadataNoFlag")); + } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataWithDataTableTest() + { + CacheMetadataWithDataTable.Test(_connStr, AddGuid("SqlBulkCopyTest_CacheMetadataDT")); + } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataCombinedWithKeepNullsTest() + { + CacheMetadataCombinedWithKeepNulls.Test(_connStr, AddGuid("SqlBulkCopyTest_CacheMetadataKeepNulls")); + } } } From a3075fbc1910e17a47cf4a7824a91745e0d240fd Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Mon, 9 Feb 2026 10:47:24 -0800 Subject: [PATCH 02/15] Fix references --- .../netcore/ref/Microsoft.Data.SqlClient.cs | 4 ++++ .../netfx/ref/Microsoft.Data.SqlClient.cs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/Microsoft.Data.SqlClient/netcore/ref/Microsoft.Data.SqlClient.cs b/src/Microsoft.Data.SqlClient/netcore/ref/Microsoft.Data.SqlClient.cs index 3cad874d58..9887e72f7b 100644 --- a/src/Microsoft.Data.SqlClient/netcore/ref/Microsoft.Data.SqlClient.cs +++ b/src/Microsoft.Data.SqlClient/netcore/ref/Microsoft.Data.SqlClient.cs @@ -315,6 +315,8 @@ public SqlBulkCopy(string connectionString, Microsoft.Data.SqlClient.SqlBulkCopy public event Microsoft.Data.SqlClient.SqlRowsCopiedEventHandler SqlRowsCopied { add { } remove { } } /// public void Close() { } + /// + public void InvalidateMetadataCache() { } /// void System.IDisposable.Dispose() { } /// @@ -441,6 +443,8 @@ public enum SqlBulkCopyOptions { /// AllowEncryptedValueModifications = 64, + /// + CacheMetadata = 128, /// CheckConstraints = 2, /// diff --git a/src/Microsoft.Data.SqlClient/netfx/ref/Microsoft.Data.SqlClient.cs b/src/Microsoft.Data.SqlClient/netfx/ref/Microsoft.Data.SqlClient.cs index 1647dfe94c..d0cd27d469 100644 --- a/src/Microsoft.Data.SqlClient/netfx/ref/Microsoft.Data.SqlClient.cs +++ b/src/Microsoft.Data.SqlClient/netfx/ref/Microsoft.Data.SqlClient.cs @@ -233,6 +233,8 @@ public SqlBulkCopy(string connectionString, Microsoft.Data.SqlClient.SqlBulkCopy public event Microsoft.Data.SqlClient.SqlRowsCopiedEventHandler SqlRowsCopied { add { } remove { } } /// public void Close() { } + /// + public void InvalidateMetadataCache() { } /// void System.IDisposable.Dispose() { } /// @@ -359,6 +361,8 @@ public enum SqlBulkCopyOptions { /// AllowEncryptedValueModifications = 64, + /// + CacheMetadata = 128, /// CheckConstraints = 2, /// From 638a2ab435b52c2229af128de81adef110bda86e Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Mon, 9 Feb 2026 12:16:22 -0800 Subject: [PATCH 03/15] Feedback Changes --- .../SqlBulkCopyOptions.xml | 1 + ....Data.SqlClient.ManualTesting.Tests.csproj | 2 +- .../SQL/SqlBulkCopyTest/CacheMetadata.cs | 76 ++++++++++++++++++- .../SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs | 6 ++ 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml index 79c3ef5c3a..120b54df8e 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml @@ -78,6 +78,7 @@ To see how the option changes the way the bulk load works, run the sample with t The cache is automatically invalidated when is changed to a different table. + Changing between operations does not require cache invalidation because the cached metadata describes only the destination table schema, not the source-to-destination column mapping. diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj index 53f0767000..118f94b5af 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj @@ -311,7 +311,7 @@ Utf8String - + diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs index 551a9512d9..dc9361d3f9 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs @@ -233,7 +233,7 @@ public static void Test(string dstConstr, string dstTable) { string initialQuery = string.Format(initialQueryTemplate, dstTable); - DataTable sourceData = new(); + using DataTable sourceData = new(); sourceData.Columns.Add("col1", typeof(int)); sourceData.Columns.Add("col2", typeof(string)); sourceData.Columns.Add("col3", typeof(string)); @@ -267,6 +267,78 @@ public static void Test(string dstConstr, string dstTable) } } + public class CacheMetadataColumnMappingsChange + { + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50), col3 nvarchar(50))"; + + // Test that changing ColumnMappings between WriteToServer calls works correctly with CacheMetadata. + // The cached metadata describes the destination table schema, not the column mappings, + // so modifying mappings between calls should work without cache invalidation. + public static void Test(string dstConstr, string dstTable) + { + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + using DataTable sourceData = new DataTable(); + sourceData.Columns.Add("id", typeof(int)); + sourceData.Columns.Add("firstName", typeof(string)); + sourceData.Columns.Add("lastName", typeof(string)); + sourceData.Rows.Add(1, "Alice", "Smith"); + sourceData.Rows.Add(2, "Bob", "Jones"); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable; + + // First write: map firstName -> col2, lastName -> col3. + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col2"); + bulkcopy.ColumnMappings.Add("lastName", "col3"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 2); + + // Verify first mapping: col2 should contain firstName values. + using (SqlCommand verifyCmd = new("select col2 from " + dstTable + " where col1 = 1", dstConn)) + { + object result = verifyCmd.ExecuteScalar(); + Assert.Equal("Alice", result); + } + + // Change mappings: swap col2 and col3 targets. + bulkcopy.ColumnMappings.Clear(); + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col3"); + bulkcopy.ColumnMappings.Add("lastName", "col2"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 4); + + // Verify second mapping: col3 should now contain firstName values for the new rows. + using (SqlCommand verifyCmd = new("select col3 from " + dstTable + " where col1 = 1 order by col2", dstConn)) + { + using SqlDataReader reader = verifyCmd.ExecuteReader(); + + // First row (from first write): col3 = "Smith" (lastName). + Assert.True(reader.Read()); + Assert.Equal("Smith", reader.GetString(0)); + + // Second row (from second write): col3 = "Alice" (firstName). + Assert.True(reader.Read()); + Assert.Equal("Alice", reader.GetString(0)); + } + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } + public class CacheMetadataCombinedWithKeepNulls { private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50) default 'DefaultVal', col3 nvarchar(50))"; @@ -276,7 +348,7 @@ public static void Test(string dstConstr, string dstTable) { string initialQuery = string.Format(initialQueryTemplate, dstTable); - DataTable sourceData = new(); + using DataTable sourceData = new(); sourceData.Columns.Add("col1", typeof(int)); sourceData.Columns.Add("col2", typeof(string)); sourceData.Columns.Add("col3", typeof(string)); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs index 21c24477a3..fccc97ff34 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs @@ -339,6 +339,12 @@ public void CacheMetadataWithDataTableTest() CacheMetadataWithDataTable.Test(_connStr, AddGuid("SqlBulkCopyTest_CacheMetadataDT")); } + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataColumnMappingsChangeTest() + { + CacheMetadataColumnMappingsChange.Test(_connStr, AddGuid("SqlBulkCopyTest_CacheMetadataColMap")); + } + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] public void CacheMetadataCombinedWithKeepNullsTest() { From 343ecd09904ec957f5200e129cab0fdbde89bf9e Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Mon, 9 Feb 2026 14:10:26 -0800 Subject: [PATCH 04/15] Handle column-pruning for cached data --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 20 ++- .../SQL/SqlBulkCopyTest/CacheMetadata.cs | 60 +++++++++ .../SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs | 6 + .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 126 ++++++++++++++++++ 4 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index d26a35fdbb..b1a24ab349 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -238,6 +238,10 @@ private int RowNumber // Metadata caching fields for CacheMetadata option private BulkCopySimpleResultSet _cachedMetadata; private string _cachedDestinationTableName; + // Per-operation clone of the destination table metadata, used when CacheMetadata is + // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not + // mutate the cached BulkCopySimpleResultSet. + private _SqlMetaDataSet _operationMetaData; #if DEBUG internal static bool s_setAlwaysTaskOnWrite; //when set and in DEBUG mode, TdsParser::WriteBulkCopyValue will always return a task @@ -612,7 +616,17 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i bool appendComma = false; // Loop over the metadata for each result column. + // When using cached metadata, clone the metadata set so that null-pruning of + // unmatched/rejected columns does not mutate the shared cache. Without this, + // changing ColumnMappings between WriteToServer calls (e.g. mapping fewer columns + // on the first call, then more on the second) would permanently lose metadata + // entries from the cache. _SqlMetaDataSet metaDataSet = internalResults[MetaDataResultId].MetaData; + if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata) && _cachedMetadata != null) + { + metaDataSet = metaDataSet.Clone(); + } + _operationMetaData = metaDataSet; _sortedColumnMappings = new List<_ColumnMapping>(metaDataSet.Length); for (int i = 0; i < metaDataSet.Length; i++) { @@ -909,7 +923,7 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults) { _stateObj.SetTimeoutSeconds(BulkCopyTimeout); - _SqlMetaDataSet metadataCollection = internalResults[MetaDataResultId].MetaData; + _SqlMetaDataSet metadataCollection = _operationMetaData ?? internalResults[MetaDataResultId].MetaData; _stateObj._outputMessageType = TdsEnums.MT_BULK; _parser.WriteBulkCopyMetaData(metadataCollection, _sortedColumnMappings.Count, _stateObj); } @@ -944,6 +958,7 @@ private void Dispose(bool disposing) _parser = null; _cachedMetadata = null; _cachedDestinationTableName = null; + _operationMetaData = null; try { // Just in case there is a lingering transaction (which there shouldn't be) @@ -2711,7 +2726,7 @@ private Task CopyBatchesAsyncContinued(BulkCopySimpleResultSet internalResults, // Load encryption keys now (if needed) _parser.LoadColumnEncryptionKeys( - internalResults[MetaDataResultId].MetaData, + _operationMetaData ?? internalResults[MetaDataResultId].MetaData, _connection); Task task = CopyRowsAsync(0, _savedBatchSize, cts); // This is copying 1 batch of rows and setting _hasMoreRowToCopy = true/false. @@ -3238,6 +3253,7 @@ private void ResetWriteToServerGlobalVariables() _dataTableSource = null; _dbDataReaderRowSource = null; _isAsyncBulkCopy = false; + _operationMetaData = null; _rowEnumerator = null; _rowSource = null; _rowSourceType = ValueSourceType.Unspecified; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs index dc9361d3f9..dd914f110d 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs @@ -339,6 +339,66 @@ public static void Test(string dstConstr, string dstTable) } } + public class CacheMetadataColumnSubsetChange + { + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50), col3 nvarchar(50))"; + + // Test that mapping a subset of columns on the first call, then all columns on the + // second call, works correctly with CacheMetadata. This verifies that null-pruning of + // unmatched columns in AnalyzeTargetAndCreateUpdateBulkCommand does not mutate the + // cached metadata, which would cause a NullReferenceException on the second call. + public static void Test(string dstConstr, string dstTable) + { + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + using DataTable sourceData = new DataTable(); + sourceData.Columns.Add("id", typeof(int)); + sourceData.Columns.Add("firstName", typeof(string)); + sourceData.Columns.Add("lastName", typeof(string)); + sourceData.Rows.Add(1, "Alice", "Smith"); + sourceData.Rows.Add(2, "Bob", "Jones"); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable; + + // First write: map only col1 and col2 (col3 is unmatched and will be pruned). + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col2"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 2); + + // Second write: map all three columns including col3. + // Without the clone fix, this would fail because col3 metadata was + // permanently nulled in the cache during the first call. + bulkcopy.ColumnMappings.Clear(); + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col2"); + bulkcopy.ColumnMappings.Add("lastName", "col3"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable, 3, 4); + + // Verify col3 has the expected data from the second write. + using (SqlCommand verifyCmd = new("select col3 from " + dstTable + " where col1 = 1 and col3 is not null", dstConn)) + { + object result = verifyCmd.ExecuteScalar(); + Assert.Equal("Smith", result); + } + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } + public class CacheMetadataCombinedWithKeepNulls { private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50) default 'DefaultVal', col3 nvarchar(50))"; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs index fccc97ff34..a93c9f5d47 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs @@ -345,6 +345,12 @@ public void CacheMetadataColumnMappingsChangeTest() CacheMetadataColumnMappingsChange.Test(_connStr, AddGuid("SqlBulkCopyTest_CacheMetadataColMap")); } + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataColumnSubsetChangeTest() + { + CacheMetadataColumnSubsetChange.Test(_connStr, AddGuid("SqlBulkCopyTest_CacheMetadataSubset")); + } + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] public void CacheMetadataCombinedWithKeepNullsTest() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs new file mode 100644 index 0000000000..447f77ff1c --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests +{ + /// + /// Tests that verify _SqlMetaDataSet.Clone() produces independent copies, + /// ensuring that null-pruning of unmatched columns in AnalyzeTargetAndCreateUpdateBulkCommand + /// does not corrupt the cached metadata when CacheMetadata is enabled. + /// + public class SqlBulkCopyCacheMetadataTest + { + [Fact] + public void SqlMetaDataSet_Clone_ProducesIndependentCopy() + { + // Arrange: create a metadata set with 3 columns simulating a destination table + _SqlMetaDataSet original = new _SqlMetaDataSet(3); + original[0].column = "col1"; + original[1].column = "col2"; + original[2].column = "col3"; + + // Act: clone and then null out an entry in the clone (simulating column pruning) + _SqlMetaDataSet clone = original.Clone(); + clone[2] = null; + + // Assert: the original is not affected by the mutation of the clone + Assert.NotNull(original[0]); + Assert.NotNull(original[1]); + Assert.NotNull(original[2]); + Assert.Equal("col1", original[0].column); + Assert.Equal("col2", original[1].column); + Assert.Equal("col3", original[2].column); + } + + [Fact] + public void SqlMetaDataSet_Clone_NullingMultipleEntries_OriginalRetainsAll() + { + // Arrange: simulate a table with 4 columns + _SqlMetaDataSet original = new _SqlMetaDataSet(4); + original[0].column = "id"; + original[1].column = "name"; + original[2].column = "email"; + original[3].column = "phone"; + + // Act: clone and null out entries 1 and 3 (simulating mapping only id and email) + _SqlMetaDataSet clone = original.Clone(); + clone[1] = null; + clone[3] = null; + + // Assert: clone has nulls where expected + Assert.NotNull(clone[0]); + Assert.Null(clone[1]); + Assert.NotNull(clone[2]); + Assert.Null(clone[3]); + + // Assert: original retains all entries + for (int i = 0; i < 4; i++) + { + Assert.NotNull(original[i]); + } + Assert.Equal("name", original[1].column); + Assert.Equal("phone", original[3].column); + } + + [Fact] + public void SqlMetaDataSet_Clone_RepeatedCloneAndPrune_OriginalSurvives() + { + // Arrange: simulate the scenario where multiple WriteToServer calls each + // clone and prune different subsets of columns + _SqlMetaDataSet original = new _SqlMetaDataSet(3); + original[0].column = "col1"; + original[1].column = "col2"; + original[2].column = "col3"; + + // First operation: map only col1 and col2 (prune col3) + _SqlMetaDataSet clone1 = original.Clone(); + clone1[2] = null; + + // Second operation: map only col1 and col3 (prune col2) + _SqlMetaDataSet clone2 = original.Clone(); + clone2[1] = null; + + // Third operation: map all columns (no pruning needed) + _SqlMetaDataSet clone3 = original.Clone(); + + // Assert: original is fully intact after all operations + Assert.NotNull(original[0]); + Assert.NotNull(original[1]); + Assert.NotNull(original[2]); + Assert.Equal("col1", original[0].column); + Assert.Equal("col2", original[1].column); + Assert.Equal("col3", original[2].column); + + // Assert: each clone reflects its own pruning + Assert.Null(clone1[2]); + Assert.NotNull(clone1[1]); + + Assert.Null(clone2[1]); + Assert.NotNull(clone2[2]); + + Assert.NotNull(clone3[0]); + Assert.NotNull(clone3[1]); + Assert.NotNull(clone3[2]); + } + + [Fact] + public void SqlMetaDataSet_Clone_PreservesOrdinals() + { + // Verify that cloned entries maintain correct ordinal values, + // which are used for column matching in AnalyzeTargetAndCreateUpdateBulkCommand + _SqlMetaDataSet original = new _SqlMetaDataSet(3); + original[0].column = "col1"; + original[1].column = "col2"; + original[2].column = "col3"; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.Equal(original[0].ordinal, clone[0].ordinal); + Assert.Equal(original[1].ordinal, clone[1].ordinal); + Assert.Equal(original[2].ordinal, clone[2].ordinal); + } + } +} From 98a3f901d2457b47c846c1c9c553008431060d55 Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Thu, 12 Feb 2026 10:54:03 -0800 Subject: [PATCH 05/15] Retain per-column encryption fields --- .../Data/SqlClient/TdsParserHelperClasses.cs | 5 + .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 143 ++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs index f189030d1e..2d44aea7da 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs @@ -352,6 +352,7 @@ private _SqlMetaDataSet(_SqlMetaDataSet original) _visibleColumnMap = original._visibleColumnMap; dbColumnSchema = original.dbColumnSchema; schemaTable = original.schemaTable; + cekTable = original.cekTable; if (original._metaDataArray == null) { @@ -577,6 +578,10 @@ internal virtual void CopyFrom(SqlMetaDataPriv original) xmlSchemaCollection = new SqlMetaDataXmlSchemaCollection(); xmlSchemaCollection.CopyFrom(original.xmlSchemaCollection); } + + this.isEncrypted = original.isEncrypted; + this.baseTI = original.baseTI; + this.cipherMD = original.cipherMD; } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index 447f77ff1c..1fe7e4efcb 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -122,5 +122,148 @@ public void SqlMetaDataSet_Clone_PreservesOrdinals() Assert.Equal(original[1].ordinal, clone[1].ordinal); Assert.Equal(original[2].ordinal, clone[2].ordinal); } + + [Fact] + public void SqlMetaDataSet_Clone_PreservesCekTable() + { + // Verify that cloning preserves the CEK table reference, which is needed by + // WriteCekTable in TdsParser to send encryption key entries to SQL Server. + // Without this, WriteCekTable sees cekTable == null and writes 0 CEK entries. + SqlTceCipherInfoTable cekTable = new SqlTceCipherInfoTable(2); + cekTable[0] = new SqlTceCipherInfoEntry(ordinal: 0); + cekTable[1] = new SqlTceCipherInfoEntry(ordinal: 1); + + _SqlMetaDataSet original = new _SqlMetaDataSet(2, cekTable); + original[0].column = "col1"; + original[1].column = "col2"; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.NotNull(clone.cekTable); + Assert.Same(original.cekTable, clone.cekTable); + Assert.Equal(2, clone.cekTable.Size); + } + + [Fact] + public void SqlMetaData_Clone_PreservesIsEncrypted() + { + // Verify that cloning a _SqlMetaData entry preserves the isEncrypted flag. + // WriteBulkCopyMetaData checks md.isEncrypted to set the TDS IsEncrypted flag + // and WriteCryptoMetadata checks it to decide whether to write cipher metadata. + // If lost, encrypted columns are sent as plaintext. + _SqlMetaDataSet original = new _SqlMetaDataSet(1); + original[0].column = "encrypted_col"; + original[0].isEncrypted = true; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.True(clone[0].isEncrypted); + } + + [Fact] + public void SqlMetaData_Clone_PreservesCipherMetadata() + { + // Verify that cloning preserves cipherMD, which is needed by + // WriteCryptoMetadata (for CekTableOrdinal, CipherAlgorithmId, etc.) + // and LoadColumnEncryptionKeys (to decrypt symmetric keys). + SqlTceCipherInfoEntry cekEntry = new SqlTceCipherInfoEntry(ordinal: 0); + SqlCipherMetadata cipherMD = new SqlCipherMetadata( + sqlTceCipherInfoEntry: cekEntry, + ordinal: 0, + cipherAlgorithmId: 2, + cipherAlgorithmName: "AEAD_AES_256_CBC_HMAC_SHA256", + encryptionType: 1, + normalizationRuleVersion: 1 + ); + + _SqlMetaDataSet original = new _SqlMetaDataSet(1); + original[0].column = "encrypted_col"; + original[0].isEncrypted = true; + original[0].cipherMD = cipherMD; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.NotNull(clone[0].cipherMD); + Assert.Equal(2, clone[0].cipherMD.CipherAlgorithmId); + Assert.Equal("AEAD_AES_256_CBC_HMAC_SHA256", clone[0].cipherMD.CipherAlgorithmName); + Assert.Equal(1, clone[0].cipherMD.EncryptionType); + Assert.Equal(1, clone[0].cipherMD.NormalizationRuleVersion); + } + + [Fact] + public void SqlMetaData_Clone_PreservesBaseTI() + { + // Verify that cloning preserves baseTI, which represents the plaintext + // TYPE_INFO for encrypted columns. WriteCryptoMetadata calls + // WriteTceUserTypeAndTypeInfo(md.baseTI) to send the unencrypted type info. + SqlMetaDataPriv baseTI = new SqlMetaDataPriv(); + baseTI.type = System.Data.SqlDbType.NVarChar; + baseTI.length = 100; + baseTI.precision = 0; + baseTI.scale = 0; + + _SqlMetaDataSet original = new _SqlMetaDataSet(1); + original[0].column = "encrypted_col"; + original[0].isEncrypted = true; + original[0].baseTI = baseTI; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.NotNull(clone[0].baseTI); + Assert.Equal(System.Data.SqlDbType.NVarChar, clone[0].baseTI.type); + Assert.Equal(100, clone[0].baseTI.length); + } + + [Fact] + public void SqlMetaDataSet_Clone_PreservesFullAlwaysEncryptedMetadata() + { + // End-to-end test: verify that a cloned _SqlMetaDataSet with Always Encrypted + // metadata retains all AE fields needed by the bulk copy TDS write path: + // cekTable (for WriteCekTable), isEncrypted (for flag writing), + // cipherMD (for WriteCryptoMetadata), and baseTI (for WriteTceUserTypeAndTypeInfo). + SqlTceCipherInfoEntry cekEntry = new SqlTceCipherInfoEntry(ordinal: 0); + SqlTceCipherInfoTable cekTable = new SqlTceCipherInfoTable(1); + cekTable[0] = cekEntry; + + SqlCipherMetadata cipherMD = new SqlCipherMetadata( + sqlTceCipherInfoEntry: cekEntry, + ordinal: 0, + cipherAlgorithmId: 2, + cipherAlgorithmName: "AEAD_AES_256_CBC_HMAC_SHA256", + encryptionType: 1, + normalizationRuleVersion: 1 + ); + + SqlMetaDataPriv baseTI = new SqlMetaDataPriv(); + baseTI.type = System.Data.SqlDbType.Int; + + _SqlMetaDataSet original = new _SqlMetaDataSet(2, cekTable); + original[0].column = "id"; + original[1].column = "secret"; + original[1].isEncrypted = true; + original[1].cipherMD = cipherMD; + original[1].baseTI = baseTI; + + // Clone and prune column 0 (simulating mapping only the encrypted column) + _SqlMetaDataSet clone = original.Clone(); + clone[0] = null; + + // The pruning must not affect the encrypted column's metadata + Assert.NotNull(clone[1]); + Assert.True(clone[1].isEncrypted); + Assert.NotNull(clone[1].cipherMD); + Assert.NotNull(clone[1].baseTI); + Assert.Equal(System.Data.SqlDbType.Int, clone[1].baseTI.type); + + // The cekTable must be preserved on the clone + Assert.NotNull(clone.cekTable); + Assert.Equal(1, clone.cekTable.Size); + + // The original must remain completely intact + Assert.NotNull(original[0]); + Assert.NotNull(original[1]); + Assert.NotNull(original.cekTable); + Assert.True(original[1].isEncrypted); + } } } From 832c073647624f67c7c7e0f5924e874076831318 Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Mon, 23 Feb 2026 15:56:26 -0800 Subject: [PATCH 06/15] Feedback changes --- .../Microsoft.Data.SqlClient/SqlBulkCopy.xml | 2 +- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 4 +- .../SqlBulkCopyCacheMetadataTest.cs | 97 ------ .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 278 +++++------------- .../Data/SqlClient/SqlMetaDataSetTest.cs | 269 +++++++++++++++++ 5 files changed, 342 insertions(+), 308 deletions(-) delete mode 100644 src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataSetTest.cs diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml index bf6e3c014b..6aaf224bcd 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml @@ -241,7 +241,7 @@ This code is provided to demonstrate the syntax for using **SqlBulkCopy** only. - Call this method when you know the destination table schema has changed and you want to force the next operation to refresh the metadata from the server. + Call this method when you know the destination table schema has changed and you want to force the next WriteToServer operation to refresh the metadata from the server. The cache is automatically invalidated when the property is changed to a different table name. diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index b1a24ab349..1d85dc5b43 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -236,8 +236,8 @@ private int RowNumber private SourceColumnMetadata[] _currentRowMetadata; // Metadata caching fields for CacheMetadata option - private BulkCopySimpleResultSet _cachedMetadata; - private string _cachedDestinationTableName; + internal BulkCopySimpleResultSet _cachedMetadata; + internal string _cachedDestinationTableName; // Per-operation clone of the destination table metadata, used when CacheMetadata is // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not // mutate the cached BulkCopySimpleResultSet. diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs deleted file mode 100644 index 8d20c47d64..0000000000 --- a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs +++ /dev/null @@ -1,97 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Xunit; - -namespace Microsoft.Data.SqlClient.Tests -{ - public class SqlBulkCopyCacheMetadataTest - { - [Fact] - public void CacheMetadata_FlagValue_IsCorrect() - { - Assert.Equal(1 << 7, (int)SqlBulkCopyOptions.CacheMetadata); - } - - [Fact] - public void CacheMetadata_CanBeCombinedWithOtherOptions() - { - SqlBulkCopyOptions combined = - SqlBulkCopyOptions.CacheMetadata | - SqlBulkCopyOptions.KeepIdentity | - SqlBulkCopyOptions.TableLock; - - Assert.True((combined & SqlBulkCopyOptions.CacheMetadata) == SqlBulkCopyOptions.CacheMetadata); - Assert.True((combined & SqlBulkCopyOptions.KeepIdentity) == SqlBulkCopyOptions.KeepIdentity); - Assert.True((combined & SqlBulkCopyOptions.TableLock) == SqlBulkCopyOptions.TableLock); - } - - [Fact] - public void CacheMetadata_DoesNotOverlapExistingFlags() - { - int cacheMetadataValue = (int)SqlBulkCopyOptions.CacheMetadata; - Assert.NotEqual((int)SqlBulkCopyOptions.Default, cacheMetadataValue); - Assert.NotEqual((int)SqlBulkCopyOptions.KeepIdentity, cacheMetadataValue); - Assert.NotEqual((int)SqlBulkCopyOptions.CheckConstraints, cacheMetadataValue); - Assert.NotEqual((int)SqlBulkCopyOptions.TableLock, cacheMetadataValue); - Assert.NotEqual((int)SqlBulkCopyOptions.KeepNulls, cacheMetadataValue); - Assert.NotEqual((int)SqlBulkCopyOptions.FireTriggers, cacheMetadataValue); - Assert.NotEqual((int)SqlBulkCopyOptions.UseInternalTransaction, cacheMetadataValue); - Assert.NotEqual((int)SqlBulkCopyOptions.AllowEncryptedValueModifications, cacheMetadataValue); - } - - [Fact] - public void InvalidateMetadataCache_CanBeCalledWithoutError() - { - using SqlBulkCopy bulkCopy = new(new SqlConnection()); - bulkCopy.InvalidateMetadataCache(); - } - - [Fact] - public void InvalidateMetadataCache_CanBeCalledMultipleTimes() - { - using SqlBulkCopy bulkCopy = new(new SqlConnection()); - bulkCopy.InvalidateMetadataCache(); - bulkCopy.InvalidateMetadataCache(); - bulkCopy.InvalidateMetadataCache(); - } - - [Fact] - public void InvalidateMetadataCache_WithCacheMetadataOption() - { - using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - bulkCopy.InvalidateMetadataCache(); - } - - [Fact] - public void InvalidateMetadataCache_WithoutCacheMetadataOption() - { - using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.Default, null); - bulkCopy.InvalidateMetadataCache(); - } - - [Fact] - public void DestinationTableName_Change_DoesNotThrowWithCacheMetadata() - { - using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - bulkCopy.DestinationTableName = "Table1"; - bulkCopy.DestinationTableName = "Table2"; - bulkCopy.DestinationTableName = "Table1"; - } - - [Fact] - public void Constructor_WithCacheMetadataOption_Succeeds() - { - using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - Assert.NotNull(bulkCopy); - } - - [Fact] - public void Constructor_WithCacheMetadataAndConnectionString_Succeeds() - { - using SqlBulkCopy bulkCopy = new("Server=localhost", SqlBulkCopyOptions.CacheMetadata); - Assert.NotNull(bulkCopy); - } - } -} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index 1fe7e4efcb..8ed1b28f06 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -2,268 +2,130 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; +using System.Linq; using Xunit; namespace Microsoft.Data.SqlClient.UnitTests { - /// - /// Tests that verify _SqlMetaDataSet.Clone() produces independent copies, - /// ensuring that null-pruning of unmatched columns in AnalyzeTargetAndCreateUpdateBulkCommand - /// does not corrupt the cached metadata when CacheMetadata is enabled. - /// public class SqlBulkCopyCacheMetadataTest { [Fact] - public void SqlMetaDataSet_Clone_ProducesIndependentCopy() + public void CacheMetadata_FlagValue_IsCorrect() { - // Arrange: create a metadata set with 3 columns simulating a destination table - _SqlMetaDataSet original = new _SqlMetaDataSet(3); - original[0].column = "col1"; - original[1].column = "col2"; - original[2].column = "col3"; - - // Act: clone and then null out an entry in the clone (simulating column pruning) - _SqlMetaDataSet clone = original.Clone(); - clone[2] = null; - - // Assert: the original is not affected by the mutation of the clone - Assert.NotNull(original[0]); - Assert.NotNull(original[1]); - Assert.NotNull(original[2]); - Assert.Equal("col1", original[0].column); - Assert.Equal("col2", original[1].column); - Assert.Equal("col3", original[2].column); + Assert.Equal(1 << 7, (int)SqlBulkCopyOptions.CacheMetadata); } [Fact] - public void SqlMetaDataSet_Clone_NullingMultipleEntries_OriginalRetainsAll() + public void CacheMetadata_CanBeCombinedWithOtherOptions() { - // Arrange: simulate a table with 4 columns - _SqlMetaDataSet original = new _SqlMetaDataSet(4); - original[0].column = "id"; - original[1].column = "name"; - original[2].column = "email"; - original[3].column = "phone"; - - // Act: clone and null out entries 1 and 3 (simulating mapping only id and email) - _SqlMetaDataSet clone = original.Clone(); - clone[1] = null; - clone[3] = null; - - // Assert: clone has nulls where expected - Assert.NotNull(clone[0]); - Assert.Null(clone[1]); - Assert.NotNull(clone[2]); - Assert.Null(clone[3]); - - // Assert: original retains all entries - for (int i = 0; i < 4; i++) - { - Assert.NotNull(original[i]); - } - Assert.Equal("name", original[1].column); - Assert.Equal("phone", original[3].column); + SqlBulkCopyOptions combined = + SqlBulkCopyOptions.CacheMetadata | + SqlBulkCopyOptions.KeepIdentity | + SqlBulkCopyOptions.TableLock; + + Assert.True((combined & SqlBulkCopyOptions.CacheMetadata) == SqlBulkCopyOptions.CacheMetadata); + Assert.True((combined & SqlBulkCopyOptions.KeepIdentity) == SqlBulkCopyOptions.KeepIdentity); + Assert.True((combined & SqlBulkCopyOptions.TableLock) == SqlBulkCopyOptions.TableLock); } [Fact] - public void SqlMetaDataSet_Clone_RepeatedCloneAndPrune_OriginalSurvives() + public void SqlBulkCopyOptions_AllValues_AreUnique() { - // Arrange: simulate the scenario where multiple WriteToServer calls each - // clone and prune different subsets of columns - _SqlMetaDataSet original = new _SqlMetaDataSet(3); - original[0].column = "col1"; - original[1].column = "col2"; - original[2].column = "col3"; - - // First operation: map only col1 and col2 (prune col3) - _SqlMetaDataSet clone1 = original.Clone(); - clone1[2] = null; - - // Second operation: map only col1 and col3 (prune col2) - _SqlMetaDataSet clone2 = original.Clone(); - clone2[1] = null; - - // Third operation: map all columns (no pruning needed) - _SqlMetaDataSet clone3 = original.Clone(); - - // Assert: original is fully intact after all operations - Assert.NotNull(original[0]); - Assert.NotNull(original[1]); - Assert.NotNull(original[2]); - Assert.Equal("col1", original[0].column); - Assert.Equal("col2", original[1].column); - Assert.Equal("col3", original[2].column); - - // Assert: each clone reflects its own pruning - Assert.Null(clone1[2]); - Assert.NotNull(clone1[1]); - - Assert.Null(clone2[1]); - Assert.NotNull(clone2[2]); + int[] values = Enum.GetValues(typeof(SqlBulkCopyOptions)) + .Cast() + .ToArray(); - Assert.NotNull(clone3[0]); - Assert.NotNull(clone3[1]); - Assert.NotNull(clone3[2]); + Assert.Equal(values.Length, values.Distinct().Count()); } [Fact] - public void SqlMetaDataSet_Clone_PreservesOrdinals() + public void InvalidateMetadataCache_ClearsCachedMetadata() { - // Verify that cloned entries maintain correct ordinal values, - // which are used for column matching in AnalyzeTargetAndCreateUpdateBulkCommand - _SqlMetaDataSet original = new _SqlMetaDataSet(3); - original[0].column = "col1"; - original[1].column = "col2"; - original[2].column = "col3"; + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - _SqlMetaDataSet clone = original.Clone(); + // Simulate cached state + bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + bulkCopy._cachedDestinationTableName = "TestTable"; - Assert.Equal(original[0].ordinal, clone[0].ordinal); - Assert.Equal(original[1].ordinal, clone[1].ordinal); - Assert.Equal(original[2].ordinal, clone[2].ordinal); + bulkCopy.InvalidateMetadataCache(); + + Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy._cachedDestinationTableName); } [Fact] - public void SqlMetaDataSet_Clone_PreservesCekTable() + public void InvalidateMetadataCache_CanBeCalledMultipleTimes() { - // Verify that cloning preserves the CEK table reference, which is needed by - // WriteCekTable in TdsParser to send encryption key entries to SQL Server. - // Without this, WriteCekTable sees cekTable == null and writes 0 CEK entries. - SqlTceCipherInfoTable cekTable = new SqlTceCipherInfoTable(2); - cekTable[0] = new SqlTceCipherInfoEntry(ordinal: 0); - cekTable[1] = new SqlTceCipherInfoEntry(ordinal: 1); + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - _SqlMetaDataSet original = new _SqlMetaDataSet(2, cekTable); - original[0].column = "col1"; - original[1].column = "col2"; + bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + bulkCopy._cachedDestinationTableName = "TestTable"; - _SqlMetaDataSet clone = original.Clone(); + bulkCopy.InvalidateMetadataCache(); + bulkCopy.InvalidateMetadataCache(); + bulkCopy.InvalidateMetadataCache(); - Assert.NotNull(clone.cekTable); - Assert.Same(original.cekTable, clone.cekTable); - Assert.Equal(2, clone.cekTable.Size); + Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy._cachedDestinationTableName); } [Fact] - public void SqlMetaData_Clone_PreservesIsEncrypted() + public void InvalidateMetadataCache_WhenNoCachedData_DoesNotThrow() { - // Verify that cloning a _SqlMetaData entry preserves the isEncrypted flag. - // WriteBulkCopyMetaData checks md.isEncrypted to set the TDS IsEncrypted flag - // and WriteCryptoMetadata checks it to decide whether to write cipher metadata. - // If lost, encrypted columns are sent as plaintext. - _SqlMetaDataSet original = new _SqlMetaDataSet(1); - original[0].column = "encrypted_col"; - original[0].isEncrypted = true; + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); + + Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy._cachedDestinationTableName); - _SqlMetaDataSet clone = original.Clone(); + bulkCopy.InvalidateMetadataCache(); - Assert.True(clone[0].isEncrypted); + Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy._cachedDestinationTableName); } [Fact] - public void SqlMetaData_Clone_PreservesCipherMetadata() + public void InvalidateMetadataCache_WithoutCacheMetadataOption_ClearsCachedMetadata() { - // Verify that cloning preserves cipherMD, which is needed by - // WriteCryptoMetadata (for CekTableOrdinal, CipherAlgorithmId, etc.) - // and LoadColumnEncryptionKeys (to decrypt symmetric keys). - SqlTceCipherInfoEntry cekEntry = new SqlTceCipherInfoEntry(ordinal: 0); - SqlCipherMetadata cipherMD = new SqlCipherMetadata( - sqlTceCipherInfoEntry: cekEntry, - ordinal: 0, - cipherAlgorithmId: 2, - cipherAlgorithmName: "AEAD_AES_256_CBC_HMAC_SHA256", - encryptionType: 1, - normalizationRuleVersion: 1 - ); + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.Default, null); - _SqlMetaDataSet original = new _SqlMetaDataSet(1); - original[0].column = "encrypted_col"; - original[0].isEncrypted = true; - original[0].cipherMD = cipherMD; + bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + bulkCopy._cachedDestinationTableName = "TestTable"; - _SqlMetaDataSet clone = original.Clone(); + bulkCopy.InvalidateMetadataCache(); - Assert.NotNull(clone[0].cipherMD); - Assert.Equal(2, clone[0].cipherMD.CipherAlgorithmId); - Assert.Equal("AEAD_AES_256_CBC_HMAC_SHA256", clone[0].cipherMD.CipherAlgorithmName); - Assert.Equal(1, clone[0].cipherMD.EncryptionType); - Assert.Equal(1, clone[0].cipherMD.NormalizationRuleVersion); + Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy._cachedDestinationTableName); } [Fact] - public void SqlMetaData_Clone_PreservesBaseTI() + public void DestinationTableName_Change_InvalidatesCachedMetadata() { - // Verify that cloning preserves baseTI, which represents the plaintext - // TYPE_INFO for encrypted columns. WriteCryptoMetadata calls - // WriteTceUserTypeAndTypeInfo(md.baseTI) to send the unencrypted type info. - SqlMetaDataPriv baseTI = new SqlMetaDataPriv(); - baseTI.type = System.Data.SqlDbType.NVarChar; - baseTI.length = 100; - baseTI.precision = 0; - baseTI.scale = 0; + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - _SqlMetaDataSet original = new _SqlMetaDataSet(1); - original[0].column = "encrypted_col"; - original[0].isEncrypted = true; - original[0].baseTI = baseTI; + bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + bulkCopy._cachedDestinationTableName = "Table1"; + bulkCopy.DestinationTableName = "Table1"; - _SqlMetaDataSet clone = original.Clone(); + // Changing to a different table should clear the cache + bulkCopy.DestinationTableName = "Table2"; - Assert.NotNull(clone[0].baseTI); - Assert.Equal(System.Data.SqlDbType.NVarChar, clone[0].baseTI.type); - Assert.Equal(100, clone[0].baseTI.length); + Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy._cachedDestinationTableName); } [Fact] - public void SqlMetaDataSet_Clone_PreservesFullAlwaysEncryptedMetadata() + public void Constructor_WithCacheMetadataOption_Succeeds() { - // End-to-end test: verify that a cloned _SqlMetaDataSet with Always Encrypted - // metadata retains all AE fields needed by the bulk copy TDS write path: - // cekTable (for WriteCekTable), isEncrypted (for flag writing), - // cipherMD (for WriteCryptoMetadata), and baseTI (for WriteTceUserTypeAndTypeInfo). - SqlTceCipherInfoEntry cekEntry = new SqlTceCipherInfoEntry(ordinal: 0); - SqlTceCipherInfoTable cekTable = new SqlTceCipherInfoTable(1); - cekTable[0] = cekEntry; - - SqlCipherMetadata cipherMD = new SqlCipherMetadata( - sqlTceCipherInfoEntry: cekEntry, - ordinal: 0, - cipherAlgorithmId: 2, - cipherAlgorithmName: "AEAD_AES_256_CBC_HMAC_SHA256", - encryptionType: 1, - normalizationRuleVersion: 1 - ); - - SqlMetaDataPriv baseTI = new SqlMetaDataPriv(); - baseTI.type = System.Data.SqlDbType.Int; - - _SqlMetaDataSet original = new _SqlMetaDataSet(2, cekTable); - original[0].column = "id"; - original[1].column = "secret"; - original[1].isEncrypted = true; - original[1].cipherMD = cipherMD; - original[1].baseTI = baseTI; - - // Clone and prune column 0 (simulating mapping only the encrypted column) - _SqlMetaDataSet clone = original.Clone(); - clone[0] = null; - - // The pruning must not affect the encrypted column's metadata - Assert.NotNull(clone[1]); - Assert.True(clone[1].isEncrypted); - Assert.NotNull(clone[1].cipherMD); - Assert.NotNull(clone[1].baseTI); - Assert.Equal(System.Data.SqlDbType.Int, clone[1].baseTI.type); - - // The cekTable must be preserved on the clone - Assert.NotNull(clone.cekTable); - Assert.Equal(1, clone.cekTable.Size); + using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); + Assert.NotNull(bulkCopy); + } - // The original must remain completely intact - Assert.NotNull(original[0]); - Assert.NotNull(original[1]); - Assert.NotNull(original.cekTable); - Assert.True(original[1].isEncrypted); + [Fact] + public void Constructor_WithCacheMetadataAndConnectionString_Succeeds() + { + using SqlBulkCopy bulkCopy = new("Server=localhost", SqlBulkCopyOptions.CacheMetadata); + Assert.NotNull(bulkCopy); } } -} +} \ No newline at end of file diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataSetTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataSetTest.cs new file mode 100644 index 0000000000..37ce70d2c8 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataSetTest.cs @@ -0,0 +1,269 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests +{ + /// + /// Tests that verify _SqlMetaDataSet.Clone() produces independent copies, + /// ensuring that null-pruning of unmatched columns in AnalyzeTargetAndCreateUpdateBulkCommand + /// does not corrupt the cached metadata when CacheMetadata is enabled. + /// + public class SqlMetaDataSetTest + { + [Fact] + public void SqlMetaDataSet_Clone_ProducesIndependentCopy() + { + // Arrange: create a metadata set with 3 columns simulating a destination table + _SqlMetaDataSet original = new _SqlMetaDataSet(3); + original[0].column = "col1"; + original[1].column = "col2"; + original[2].column = "col3"; + + // Act: clone and then null out an entry in the clone (simulating column pruning) + _SqlMetaDataSet clone = original.Clone(); + clone[2] = null; + + // Assert: the original is not affected by the mutation of the clone + Assert.NotNull(original[0]); + Assert.NotNull(original[1]); + Assert.NotNull(original[2]); + Assert.Equal("col1", original[0].column); + Assert.Equal("col2", original[1].column); + Assert.Equal("col3", original[2].column); + } + + [Fact] + public void SqlMetaDataSet_Clone_NullingMultipleEntries_OriginalRetainsAll() + { + // Arrange: simulate a table with 4 columns + _SqlMetaDataSet original = new _SqlMetaDataSet(4); + original[0].column = "id"; + original[1].column = "name"; + original[2].column = "email"; + original[3].column = "phone"; + + // Act: clone and null out entries 1 and 3 (simulating mapping only id and email) + _SqlMetaDataSet clone = original.Clone(); + clone[1] = null; + clone[3] = null; + + // Assert: clone has nulls where expected + Assert.NotNull(clone[0]); + Assert.Null(clone[1]); + Assert.NotNull(clone[2]); + Assert.Null(clone[3]); + + // Assert: original retains all entries + for (int i = 0; i < 4; i++) + { + Assert.NotNull(original[i]); + } + Assert.Equal("name", original[1].column); + Assert.Equal("phone", original[3].column); + } + + [Fact] + public void SqlMetaDataSet_Clone_RepeatedCloneAndPrune_OriginalSurvives() + { + // Arrange: simulate the scenario where multiple WriteToServer calls each + // clone and prune different subsets of columns + _SqlMetaDataSet original = new _SqlMetaDataSet(3); + original[0].column = "col1"; + original[1].column = "col2"; + original[2].column = "col3"; + + // First operation: map only col1 and col2 (prune col3) + _SqlMetaDataSet clone1 = original.Clone(); + clone1[2] = null; + + // Second operation: map only col1 and col3 (prune col2) + _SqlMetaDataSet clone2 = original.Clone(); + clone2[1] = null; + + // Third operation: map all columns (no pruning needed) + _SqlMetaDataSet clone3 = original.Clone(); + + // Assert: original is fully intact after all operations + Assert.NotNull(original[0]); + Assert.NotNull(original[1]); + Assert.NotNull(original[2]); + Assert.Equal("col1", original[0].column); + Assert.Equal("col2", original[1].column); + Assert.Equal("col3", original[2].column); + + // Assert: each clone reflects its own pruning + Assert.Null(clone1[2]); + Assert.NotNull(clone1[1]); + + Assert.Null(clone2[1]); + Assert.NotNull(clone2[2]); + + Assert.NotNull(clone3[0]); + Assert.NotNull(clone3[1]); + Assert.NotNull(clone3[2]); + } + + [Fact] + public void SqlMetaDataSet_Clone_PreservesOrdinals() + { + // Verify that cloned entries maintain correct ordinal values, + // which are used for column matching in AnalyzeTargetAndCreateUpdateBulkCommand + _SqlMetaDataSet original = new _SqlMetaDataSet(3); + original[0].column = "col1"; + original[1].column = "col2"; + original[2].column = "col3"; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.Equal(original[0].ordinal, clone[0].ordinal); + Assert.Equal(original[1].ordinal, clone[1].ordinal); + Assert.Equal(original[2].ordinal, clone[2].ordinal); + } + + [Fact] + public void SqlMetaDataSet_Clone_PreservesCekTable() + { + // Verify that cloning preserves the CEK table reference, which is needed by + // WriteCekTable in TdsParser to send encryption key entries to SQL Server. + // Without this, WriteCekTable sees cekTable == null and writes 0 CEK entries. + SqlTceCipherInfoTable cekTable = new SqlTceCipherInfoTable(2); + cekTable[0] = new SqlTceCipherInfoEntry(ordinal: 0); + cekTable[1] = new SqlTceCipherInfoEntry(ordinal: 1); + + _SqlMetaDataSet original = new _SqlMetaDataSet(2, cekTable); + original[0].column = "col1"; + original[1].column = "col2"; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.NotNull(clone.cekTable); + Assert.Same(original.cekTable, clone.cekTable); + Assert.Equal(2, clone.cekTable.Size); + } + + [Fact] + public void SqlMetaData_Clone_PreservesIsEncrypted() + { + // Verify that cloning a _SqlMetaData entry preserves the isEncrypted flag. + // WriteBulkCopyMetaData checks md.isEncrypted to set the TDS IsEncrypted flag + // and WriteCryptoMetadata checks it to decide whether to write cipher metadata. + // If lost, encrypted columns are sent as plaintext. + _SqlMetaDataSet original = new _SqlMetaDataSet(1); + original[0].column = "encrypted_col"; + original[0].isEncrypted = true; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.True(clone[0].isEncrypted); + } + + [Fact] + public void SqlMetaData_Clone_PreservesCipherMetadata() + { + // Verify that cloning preserves cipherMD, which is needed by + // WriteCryptoMetadata (for CekTableOrdinal, CipherAlgorithmId, etc.) + // and LoadColumnEncryptionKeys (to decrypt symmetric keys). + SqlTceCipherInfoEntry cekEntry = new SqlTceCipherInfoEntry(ordinal: 0); + SqlCipherMetadata cipherMD = new SqlCipherMetadata( + sqlTceCipherInfoEntry: cekEntry, + ordinal: 0, + cipherAlgorithmId: 2, + cipherAlgorithmName: "AEAD_AES_256_CBC_HMAC_SHA256", + encryptionType: 1, + normalizationRuleVersion: 1 + ); + + _SqlMetaDataSet original = new _SqlMetaDataSet(1); + original[0].column = "encrypted_col"; + original[0].isEncrypted = true; + original[0].cipherMD = cipherMD; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.NotNull(clone[0].cipherMD); + Assert.Equal(2, clone[0].cipherMD.CipherAlgorithmId); + Assert.Equal("AEAD_AES_256_CBC_HMAC_SHA256", clone[0].cipherMD.CipherAlgorithmName); + Assert.Equal(1, clone[0].cipherMD.EncryptionType); + Assert.Equal(1, clone[0].cipherMD.NormalizationRuleVersion); + } + + [Fact] + public void SqlMetaData_Clone_PreservesBaseTI() + { + // Verify that cloning preserves baseTI, which represents the plaintext + // TYPE_INFO for encrypted columns. WriteCryptoMetadata calls + // WriteTceUserTypeAndTypeInfo(md.baseTI) to send the unencrypted type info. + SqlMetaDataPriv baseTI = new SqlMetaDataPriv(); + baseTI.type = System.Data.SqlDbType.NVarChar; + baseTI.length = 100; + baseTI.precision = 0; + baseTI.scale = 0; + + _SqlMetaDataSet original = new _SqlMetaDataSet(1); + original[0].column = "encrypted_col"; + original[0].isEncrypted = true; + original[0].baseTI = baseTI; + + _SqlMetaDataSet clone = original.Clone(); + + Assert.NotNull(clone[0].baseTI); + Assert.Equal(System.Data.SqlDbType.NVarChar, clone[0].baseTI.type); + Assert.Equal(100, clone[0].baseTI.length); + } + + [Fact] + public void SqlMetaDataSet_Clone_PreservesFullAlwaysEncryptedMetadata() + { + // End-to-end test: verify that a cloned _SqlMetaDataSet with Always Encrypted + // metadata retains all AE fields needed by the bulk copy TDS write path: + // cekTable (for WriteCekTable), isEncrypted (for flag writing), + // cipherMD (for WriteCryptoMetadata), and baseTI (for WriteTceUserTypeAndTypeInfo). + SqlTceCipherInfoEntry cekEntry = new SqlTceCipherInfoEntry(ordinal: 0); + SqlTceCipherInfoTable cekTable = new SqlTceCipherInfoTable(1); + cekTable[0] = cekEntry; + + SqlCipherMetadata cipherMD = new SqlCipherMetadata( + sqlTceCipherInfoEntry: cekEntry, + ordinal: 0, + cipherAlgorithmId: 2, + cipherAlgorithmName: "AEAD_AES_256_CBC_HMAC_SHA256", + encryptionType: 1, + normalizationRuleVersion: 1 + ); + + SqlMetaDataPriv baseTI = new SqlMetaDataPriv(); + baseTI.type = System.Data.SqlDbType.Int; + + _SqlMetaDataSet original = new _SqlMetaDataSet(2, cekTable); + original[0].column = "id"; + original[1].column = "secret"; + original[1].isEncrypted = true; + original[1].cipherMD = cipherMD; + original[1].baseTI = baseTI; + + // Clone and prune column 0 (simulating mapping only the encrypted column) + _SqlMetaDataSet clone = original.Clone(); + clone[0] = null; + + // The pruning must not affect the encrypted column's metadata + Assert.NotNull(clone[1]); + Assert.True(clone[1].isEncrypted); + Assert.NotNull(clone[1].cipherMD); + Assert.NotNull(clone[1].baseTI); + Assert.Equal(System.Data.SqlDbType.Int, clone[1].baseTI.type); + + // The cekTable must be preserved on the clone + Assert.NotNull(clone.cekTable); + Assert.Equal(1, clone.cekTable.Size); + + // The original must remain completely intact + Assert.NotNull(original[0]); + Assert.NotNull(original[1]); + Assert.NotNull(original.cekTable); + Assert.True(original[1].isEncrypted); + } + } +} \ No newline at end of file From 4b1aef7ee788b3776c35e6cf0ad19b0e3d6ba02d Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Mon, 23 Feb 2026 16:28:32 -0800 Subject: [PATCH 07/15] Nullify operationMetaData --- .../src/Microsoft/Data/SqlClient/SqlBulkCopy.cs | 4 +++- .../Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 1d85dc5b43..b6a3e62a2d 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -241,7 +241,7 @@ private int RowNumber // Per-operation clone of the destination table metadata, used when CacheMetadata is // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not // mutate the cached BulkCopySimpleResultSet. - private _SqlMetaDataSet _operationMetaData; + internal _SqlMetaDataSet _operationMetaData; #if DEBUG internal static bool s_setAlwaysTaskOnWrite; //when set and in DEBUG mode, TdsParser::WriteBulkCopyValue will always return a task @@ -367,6 +367,7 @@ public string DestinationTableName { _cachedMetadata = null; _cachedDestinationTableName = null; + _operationMetaData = null; } _destinationTableName = value; @@ -933,6 +934,7 @@ public void InvalidateMetadataCache() { _cachedMetadata = null; _cachedDestinationTableName = null; + _operationMetaData = null; SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.InvalidateMetadataCache | Info | Metadata cache invalidated"); } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index 8ed1b28f06..e34e2cadce 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -47,11 +47,13 @@ public void InvalidateMetadataCache_ClearsCachedMetadata() // Simulate cached state bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); bulkCopy._cachedDestinationTableName = "TestTable"; + bulkCopy._operationMetaData = new _SqlMetaDataSet(1); bulkCopy.InvalidateMetadataCache(); Assert.Null(bulkCopy._cachedMetadata); Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } [Fact] @@ -61,6 +63,7 @@ public void InvalidateMetadataCache_CanBeCalledMultipleTimes() bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); bulkCopy._cachedDestinationTableName = "TestTable"; + bulkCopy._operationMetaData = new _SqlMetaDataSet(1); bulkCopy.InvalidateMetadataCache(); bulkCopy.InvalidateMetadataCache(); @@ -68,6 +71,7 @@ public void InvalidateMetadataCache_CanBeCalledMultipleTimes() Assert.Null(bulkCopy._cachedMetadata); Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } [Fact] @@ -91,11 +95,13 @@ public void InvalidateMetadataCache_WithoutCacheMetadataOption_ClearsCachedMetad bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); bulkCopy._cachedDestinationTableName = "TestTable"; + bulkCopy._operationMetaData = new _SqlMetaDataSet(1); bulkCopy.InvalidateMetadataCache(); Assert.Null(bulkCopy._cachedMetadata); Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } [Fact] @@ -105,6 +111,7 @@ public void DestinationTableName_Change_InvalidatesCachedMetadata() bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); bulkCopy._cachedDestinationTableName = "Table1"; + bulkCopy._operationMetaData = new _SqlMetaDataSet(1); bulkCopy.DestinationTableName = "Table1"; // Changing to a different table should clear the cache @@ -112,6 +119,7 @@ public void DestinationTableName_Change_InvalidatesCachedMetadata() Assert.Null(bulkCopy._cachedMetadata); Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } [Fact] From 1b7dc5a73db1597025cc884e02f5931675dec33a Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Tue, 24 Feb 2026 12:01:18 -0800 Subject: [PATCH 08/15] Simplify caching --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 8 +----- .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 28 +++++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index b6a3e62a2d..8c0b5aea44 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -237,7 +237,6 @@ private int RowNumber // Metadata caching fields for CacheMetadata option internal BulkCopySimpleResultSet _cachedMetadata; - internal string _cachedDestinationTableName; // Per-operation clone of the destination table metadata, used when CacheMetadata is // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not // mutate the cached BulkCopySimpleResultSet. @@ -366,7 +365,6 @@ public string DestinationTableName if (!string.Equals(_destinationTableName, value, StringComparison.Ordinal)) { _cachedMetadata = null; - _cachedDestinationTableName = null; _operationMetaData = null; } @@ -516,8 +514,7 @@ private Task CreateAndExecuteInitialQueryAsync(out Bulk { // Check if we have valid cached metadata for the current destination table if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata) && - _cachedMetadata != null && - string.Equals(_cachedDestinationTableName, _destinationTableName, StringComparison.Ordinal)) + _cachedMetadata != null) { SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName); result = _cachedMetadata; @@ -563,7 +560,6 @@ private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result) if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata)) { _cachedMetadata = result; - _cachedDestinationTableName = _destinationTableName; SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CacheMetadataIfEnabled | Info | Cached metadata for table '{0}'", _destinationTableName); } } @@ -933,7 +929,6 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults) public void InvalidateMetadataCache() { _cachedMetadata = null; - _cachedDestinationTableName = null; _operationMetaData = null; SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.InvalidateMetadataCache | Info | Metadata cache invalidated"); } @@ -959,7 +954,6 @@ private void Dispose(bool disposing) _columnMappings = null; _parser = null; _cachedMetadata = null; - _cachedDestinationTableName = null; _operationMetaData = null; try { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index e34e2cadce..dae7d16cbb 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -46,13 +46,13 @@ public void InvalidateMetadataCache_ClearsCachedMetadata() // Simulate cached state bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._cachedDestinationTableName = "TestTable"; + bulkCopy._operationMetaData = new _SqlMetaDataSet(1); bulkCopy.InvalidateMetadataCache(); Assert.Null(bulkCopy._cachedMetadata); - Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } @@ -62,7 +62,7 @@ public void InvalidateMetadataCache_CanBeCalledMultipleTimes() using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._cachedDestinationTableName = "TestTable"; + bulkCopy._operationMetaData = new _SqlMetaDataSet(1); bulkCopy.InvalidateMetadataCache(); @@ -70,7 +70,7 @@ public void InvalidateMetadataCache_CanBeCalledMultipleTimes() bulkCopy.InvalidateMetadataCache(); Assert.Null(bulkCopy._cachedMetadata); - Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } @@ -80,12 +80,12 @@ public void InvalidateMetadataCache_WhenNoCachedData_DoesNotThrow() using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); Assert.Null(bulkCopy._cachedMetadata); - Assert.Null(bulkCopy._cachedDestinationTableName); + bulkCopy.InvalidateMetadataCache(); Assert.Null(bulkCopy._cachedMetadata); - Assert.Null(bulkCopy._cachedDestinationTableName); + } [Fact] @@ -94,13 +94,13 @@ public void InvalidateMetadataCache_WithoutCacheMetadataOption_ClearsCachedMetad using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.Default, null); bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._cachedDestinationTableName = "TestTable"; + bulkCopy._operationMetaData = new _SqlMetaDataSet(1); bulkCopy.InvalidateMetadataCache(); Assert.Null(bulkCopy._cachedMetadata); - Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } @@ -109,16 +109,22 @@ public void DestinationTableName_Change_InvalidatesCachedMetadata() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); + // Set backing field first so the setter sees a matching name + bulkCopy.DestinationTableName = "Table1"; + + // Simulate cached state after a WriteToServer call bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._cachedDestinationTableName = "Table1"; bulkCopy._operationMetaData = new _SqlMetaDataSet(1); + + // Setting the same name should NOT clear the cache bulkCopy.DestinationTableName = "Table1"; + Assert.NotNull(bulkCopy._cachedMetadata); + Assert.NotNull(bulkCopy._operationMetaData); // Changing to a different table should clear the cache bulkCopy.DestinationTableName = "Table2"; - Assert.Null(bulkCopy._cachedMetadata); - Assert.Null(bulkCopy._cachedDestinationTableName); + Assert.Null(bulkCopy._operationMetaData); } From 76741462492f7e78ac86ab74e56b7d66fa33a12c Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Tue, 24 Feb 2026 12:29:57 -0800 Subject: [PATCH 09/15] Rename Optional Flag --- .../Microsoft.Data.SqlClient/SqlBulkCopy.xml | 17 +++++++++---- .../SqlBulkCopyOptions.xml | 24 +++++++++++++++---- .../ref/Microsoft.Data.SqlClient.cs | 4 ++-- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 15 ++++++------ .../SQL/SqlBulkCopyTest/CacheMetadata.cs | 4 ++-- .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 20 ++++++++-------- 6 files changed, 53 insertions(+), 31 deletions(-) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml index 6aaf224bcd..740d3beee3 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml @@ -235,19 +235,26 @@ This code is provided to demonstrate the syntax for using **SqlBulkCopy** only. ]]> - + - Clears the cached destination table metadata when using the option. + Clears the cached destination table metadata when using the + + option. - Call this method when you know the destination table schema has changed and you want to force the next WriteToServer operation to refresh the metadata from the server. + Call this method when you know the destination table schema + has changed and you want to force the next + WriteToServer operation to refresh the metadata + from the server. - The cache is automatically invalidated when the property is changed to a different table name. + The cache is automatically invalidated when the + + property is changed to a different table name. - + Enables or disables a object to stream data from an object diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml index 120b54df8e..65502f158a 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml @@ -71,14 +71,30 @@ To see how the option changes the way the bulk load works, run the sample with t - When specified, CacheMetadata caches destination table metadata after the first bulk copy operation, allowing subsequent operations to the same table to skip the metadata discovery query. This can improve performance when performing multiple bulk copy operations to the same destination table. + When specified, CacheMetadata caches destination table + metadata after the first bulk copy operation, allowing + subsequent operations to the same table to skip the metadata + discovery query. This can improve performance when performing + multiple bulk copy operations to the same destination table. - Warning: Use this option only when you are certain the destination table schema will not change between bulk copy operations. If the table schema changes (columns added, removed, or modified), using cached metadata may result in data corruption, failed operations, or unexpected behavior. Call to clear the cache if the schema changes. + Warning: Use this option only when you are certain the + destination table schema will not change between bulk copy + operations. If the table schema changes (columns added, + removed, or modified), using cached metadata may result in + data corruption, failed operations, or unexpected behavior. + Call + + to clear the cache if the schema changes. - The cache is automatically invalidated when is changed to a different table. - Changing between operations does not require cache invalidation because the cached metadata describes only the destination table schema, not the source-to-destination column mapping. + The cache is automatically invalidated when + + is changed to a different table. Changing + + between operations does not require cache invalidation + because the cached metadata describes only the destination + table schema, not the source-to-destination column mapping. diff --git a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs index 43a6a61914..69a29c5b02 100644 --- a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs +++ b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs @@ -212,8 +212,8 @@ public SqlBulkCopy(string connectionString, Microsoft.Data.SqlClient.SqlBulkCopy public event Microsoft.Data.SqlClient.SqlRowsCopiedEventHandler SqlRowsCopied { add { } remove { } } /// public void Close() { } - /// - public void InvalidateMetadataCache() { } + /// + public void ClearCachedMetadata() { } /// void System.IDisposable.Dispose() { } /// diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 8c0b5aea44..834497c3bb 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -360,14 +360,13 @@ public string DestinationTableName { throw ADP.ArgumentOutOfRange(nameof(DestinationTableName)); } - - // Invalidate cached metadata if the destination table name changes - if (!string.Equals(_destinationTableName, value, StringComparison.Ordinal)) + else if (string.Equals(_destinationTableName, value, StringComparison.Ordinal)) { - _cachedMetadata = null; - _operationMetaData = null; + return; } + _cachedMetadata = null; + _operationMetaData = null; _destinationTableName = value; } } @@ -925,12 +924,12 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults) _parser.WriteBulkCopyMetaData(metadataCollection, _sortedColumnMappings.Count, _stateObj); } - /// - public void InvalidateMetadataCache() + /// + public void ClearCachedMetadata() { _cachedMetadata = null; _operationMetaData = null; - SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.InvalidateMetadataCache | Info | Metadata cache invalidated"); + SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.ClearCachedMetadata | Info | Metadata cache invalidated"); } // Terminates the bulk copy operation. diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs index dd914f110d..adc85bf3ca 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs @@ -74,7 +74,7 @@ public class CacheMetadataInvalidate private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; - // Test that InvalidateMetadataCache forces a fresh metadata query. + // Test that ClearCachedMetadata forces a fresh metadata query. public static void Test(string srcConstr, string dstConstr, string dstTable) { string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); @@ -102,7 +102,7 @@ public static void Test(string srcConstr, string dstConstr, string dstTable) Helpers.VerifyResults(dstConn, dstTable, 3, 5); // Invalidate the cache and write again: should still succeed after re-querying metadata. - bulkcopy.InvalidateMetadataCache(); + bulkcopy.ClearCachedMetadata(); using (SqlConnection srcConn = new(srcConstr)) { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index dae7d16cbb..65bf0d8e77 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -40,7 +40,7 @@ public void SqlBulkCopyOptions_AllValues_AreUnique() } [Fact] - public void InvalidateMetadataCache_ClearsCachedMetadata() + public void ClearCachedMetadata_ClearsCachedMetadata() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); @@ -49,7 +49,7 @@ public void InvalidateMetadataCache_ClearsCachedMetadata() bulkCopy._operationMetaData = new _SqlMetaDataSet(1); - bulkCopy.InvalidateMetadataCache(); + bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); @@ -57,7 +57,7 @@ public void InvalidateMetadataCache_ClearsCachedMetadata() } [Fact] - public void InvalidateMetadataCache_CanBeCalledMultipleTimes() + public void ClearCachedMetadata_CanBeCalledMultipleTimes() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); @@ -65,9 +65,9 @@ public void InvalidateMetadataCache_CanBeCalledMultipleTimes() bulkCopy._operationMetaData = new _SqlMetaDataSet(1); - bulkCopy.InvalidateMetadataCache(); - bulkCopy.InvalidateMetadataCache(); - bulkCopy.InvalidateMetadataCache(); + bulkCopy.ClearCachedMetadata(); + bulkCopy.ClearCachedMetadata(); + bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); @@ -75,21 +75,21 @@ public void InvalidateMetadataCache_CanBeCalledMultipleTimes() } [Fact] - public void InvalidateMetadataCache_WhenNoCachedData_DoesNotThrow() + public void ClearCachedMetadata_WhenNoCachedData_DoesNotThrow() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); Assert.Null(bulkCopy._cachedMetadata); - bulkCopy.InvalidateMetadataCache(); + bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); } [Fact] - public void InvalidateMetadataCache_WithoutCacheMetadataOption_ClearsCachedMetadata() + public void ClearCachedMetadata_WithoutCacheMetadataOption_ClearsCachedMetadata() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.Default, null); @@ -97,7 +97,7 @@ public void InvalidateMetadataCache_WithoutCacheMetadataOption_ClearsCachedMetad bulkCopy._operationMetaData = new _SqlMetaDataSet(1); - bulkCopy.InvalidateMetadataCache(); + bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); From c06db4c279dd5d61a80356b0b34fc2aea9c5b74b Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Tue, 24 Feb 2026 15:22:40 -0800 Subject: [PATCH 10/15] Fix _operationMetaData --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 6 ++--- .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 23 ++----------------- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 834497c3bb..6ac17b4520 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -240,7 +240,7 @@ private int RowNumber // Per-operation clone of the destination table metadata, used when CacheMetadata is // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not // mutate the cached BulkCopySimpleResultSet. - internal _SqlMetaDataSet _operationMetaData; + private _SqlMetaDataSet _operationMetaData; #if DEBUG internal static bool s_setAlwaysTaskOnWrite; //when set and in DEBUG mode, TdsParser::WriteBulkCopyValue will always return a task @@ -366,7 +366,6 @@ public string DestinationTableName } _cachedMetadata = null; - _operationMetaData = null; _destinationTableName = value; } } @@ -928,8 +927,7 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults) public void ClearCachedMetadata() { _cachedMetadata = null; - _operationMetaData = null; - SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.ClearCachedMetadata | Info | Metadata cache invalidated"); + SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.ClearCachedMetadata | Info | Metadata cache cleared"); } // Terminates the bulk copy operation. diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index 65bf0d8e77..5027651dc9 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -44,16 +44,11 @@ public void ClearCachedMetadata_ClearsCachedMetadata() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - // Simulate cached state bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._operationMetaData = new _SqlMetaDataSet(1); - bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); - - Assert.Null(bulkCopy._operationMetaData); } [Fact] @@ -63,15 +58,11 @@ public void ClearCachedMetadata_CanBeCalledMultipleTimes() bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._operationMetaData = new _SqlMetaDataSet(1); - bulkCopy.ClearCachedMetadata(); bulkCopy.ClearCachedMetadata(); bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); - - Assert.Null(bulkCopy._operationMetaData); } [Fact] @@ -81,11 +72,9 @@ public void ClearCachedMetadata_WhenNoCachedData_DoesNotThrow() Assert.Null(bulkCopy._cachedMetadata); - bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); - } [Fact] @@ -95,17 +84,13 @@ public void ClearCachedMetadata_WithoutCacheMetadataOption_ClearsCachedMetadata( bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._operationMetaData = new _SqlMetaDataSet(1); - bulkCopy.ClearCachedMetadata(); Assert.Null(bulkCopy._cachedMetadata); - - Assert.Null(bulkCopy._operationMetaData); } [Fact] - public void DestinationTableName_Change_InvalidatesCachedMetadata() + public void DestinationTableName_Change_ClearsCachedMetadata() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); @@ -114,18 +99,14 @@ public void DestinationTableName_Change_InvalidatesCachedMetadata() // Simulate cached state after a WriteToServer call bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); - bulkCopy._operationMetaData = new _SqlMetaDataSet(1); // Setting the same name should NOT clear the cache bulkCopy.DestinationTableName = "Table1"; Assert.NotNull(bulkCopy._cachedMetadata); - Assert.NotNull(bulkCopy._operationMetaData); // Changing to a different table should clear the cache bulkCopy.DestinationTableName = "Table2"; Assert.Null(bulkCopy._cachedMetadata); - - Assert.Null(bulkCopy._operationMetaData); } [Fact] @@ -142,4 +123,4 @@ public void Constructor_WithCacheMetadataAndConnectionString_Succeeds() Assert.NotNull(bulkCopy); } } -} \ No newline at end of file +} From 55c2ed9db2c6162d557469e4ce120875c381fc13 Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Wed, 25 Feb 2026 10:53:17 -0800 Subject: [PATCH 11/15] Remove redundant check --- .../src/Microsoft/Data/SqlClient/SqlBulkCopy.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 6ac17b4520..8c1bb992ed 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -511,8 +511,7 @@ IF EXISTS (SELECT TOP 1 * FROM sys.all_columns WHERE [object_id] = OBJECT_ID('sy private Task CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result) { // Check if we have valid cached metadata for the current destination table - if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata) && - _cachedMetadata != null) + if (_cachedMetadata != null) { SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName); result = _cachedMetadata; @@ -617,7 +616,7 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i // on the first call, then more on the second) would permanently lose metadata // entries from the cache. _SqlMetaDataSet metaDataSet = internalResults[MetaDataResultId].MetaData; - if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata) && _cachedMetadata != null) + if (_cachedMetadata != null) { metaDataSet = metaDataSet.Clone(); } From 261761abb36459f02bcac3f4d02b982e1a09a4c3 Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Wed, 25 Feb 2026 11:23:44 -0800 Subject: [PATCH 12/15] Update docs --- doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml | 9 +++++++++ .../Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml index 740d3beee3..5e8f58d414 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml @@ -253,6 +253,15 @@ This code is provided to demonstrate the syntax for using **SqlBulkCopy** only. property is changed to a different table name. + + The cache is not automatically invalidated when the + connection context changes. Call this method if the + underlying + + changes database (for example, via + ) + or reconnects to a different server due to failover. + diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml index 65502f158a..c49537e4d1 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml @@ -96,6 +96,17 @@ To see how the option changes the way the bulk load works, run the sample with t because the cached metadata describes only the destination table schema, not the source-to-destination column mapping. + + The cache is not automatically invalidated when the + connection context changes. If the underlying + + changes database (for example, via + ) + or reconnects to a different server due to failover, callers + should call + + to ensure the metadata is refreshed. + From a4bf542f86ebe7f6ff9daedc613e141a9a42866a Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Wed, 25 Feb 2026 15:12:01 -0800 Subject: [PATCH 13/15] Update CachedMetadata property --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 16 +++++----- .../SqlClient/SqlBulkCopyCacheMetadataTest.cs | 30 ++++++++++++------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 8c1bb992ed..048a90efb6 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -236,7 +236,7 @@ private int RowNumber private SourceColumnMetadata[] _currentRowMetadata; // Metadata caching fields for CacheMetadata option - internal BulkCopySimpleResultSet _cachedMetadata; + internal BulkCopySimpleResultSet CachedMetadata { get; private set; } // Per-operation clone of the destination table metadata, used when CacheMetadata is // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not // mutate the cached BulkCopySimpleResultSet. @@ -365,7 +365,7 @@ public string DestinationTableName return; } - _cachedMetadata = null; + CachedMetadata = null; _destinationTableName = value; } } @@ -511,10 +511,10 @@ IF EXISTS (SELECT TOP 1 * FROM sys.all_columns WHERE [object_id] = OBJECT_ID('sy private Task CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result) { // Check if we have valid cached metadata for the current destination table - if (_cachedMetadata != null) + if (CachedMetadata != null) { SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName); - result = _cachedMetadata; + result = CachedMetadata; return null; } @@ -556,7 +556,7 @@ private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result) { if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata)) { - _cachedMetadata = result; + CachedMetadata = result; SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CacheMetadataIfEnabled | Info | Cached metadata for table '{0}'", _destinationTableName); } } @@ -616,7 +616,7 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i // on the first call, then more on the second) would permanently lose metadata // entries from the cache. _SqlMetaDataSet metaDataSet = internalResults[MetaDataResultId].MetaData; - if (_cachedMetadata != null) + if (CachedMetadata != null) { metaDataSet = metaDataSet.Clone(); } @@ -925,7 +925,7 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults) /// public void ClearCachedMetadata() { - _cachedMetadata = null; + CachedMetadata = null; SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.ClearCachedMetadata | Info | Metadata cache cleared"); } @@ -949,7 +949,7 @@ private void Dispose(bool disposing) // Dispose dependent objects _columnMappings = null; _parser = null; - _cachedMetadata = null; + CachedMetadata = null; _operationMetaData = null; try { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs index 5027651dc9..90aa4a8f5e 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlBulkCopyCacheMetadataTest.cs @@ -4,12 +4,20 @@ using System; using System.Linq; +using System.Reflection; using Xunit; namespace Microsoft.Data.SqlClient.UnitTests { public class SqlBulkCopyCacheMetadataTest { + private static void SetCachedMetadata(SqlBulkCopy bulkCopy, BulkCopySimpleResultSet value) + { + typeof(SqlBulkCopy) + .GetProperty("CachedMetadata", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public)! + .SetValue(bulkCopy, value); + } + [Fact] public void CacheMetadata_FlagValue_IsCorrect() { @@ -44,11 +52,11 @@ public void ClearCachedMetadata_ClearsCachedMetadata() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + SetCachedMetadata(bulkCopy, new BulkCopySimpleResultSet()); bulkCopy.ClearCachedMetadata(); - Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy.CachedMetadata); } [Fact] @@ -56,13 +64,13 @@ public void ClearCachedMetadata_CanBeCalledMultipleTimes() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + SetCachedMetadata(bulkCopy, new BulkCopySimpleResultSet()); bulkCopy.ClearCachedMetadata(); bulkCopy.ClearCachedMetadata(); bulkCopy.ClearCachedMetadata(); - Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy.CachedMetadata); } [Fact] @@ -70,11 +78,11 @@ public void ClearCachedMetadata_WhenNoCachedData_DoesNotThrow() { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null); - Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy.CachedMetadata); bulkCopy.ClearCachedMetadata(); - Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy.CachedMetadata); } [Fact] @@ -82,11 +90,11 @@ public void ClearCachedMetadata_WithoutCacheMetadataOption_ClearsCachedMetadata( { using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.Default, null); - bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + SetCachedMetadata(bulkCopy, new BulkCopySimpleResultSet()); bulkCopy.ClearCachedMetadata(); - Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy.CachedMetadata); } [Fact] @@ -98,15 +106,15 @@ public void DestinationTableName_Change_ClearsCachedMetadata() bulkCopy.DestinationTableName = "Table1"; // Simulate cached state after a WriteToServer call - bulkCopy._cachedMetadata = new BulkCopySimpleResultSet(); + SetCachedMetadata(bulkCopy, new BulkCopySimpleResultSet()); // Setting the same name should NOT clear the cache bulkCopy.DestinationTableName = "Table1"; - Assert.NotNull(bulkCopy._cachedMetadata); + Assert.NotNull(bulkCopy.CachedMetadata); // Changing to a different table should clear the cache bulkCopy.DestinationTableName = "Table2"; - Assert.Null(bulkCopy._cachedMetadata); + Assert.Null(bulkCopy.CachedMetadata); } [Fact] From 2663cf2af54542db58061a892b60c80be3be2a98 Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Wed, 25 Feb 2026 16:00:03 -0800 Subject: [PATCH 14/15] Add async tests --- .../SQL/SqlBulkCopyTest/CacheMetadata.cs | 68 +++++++++++++++++++ .../SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs | 6 ++ 2 files changed, 74 insertions(+) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs index adc85bf3ca..36f37fc2e7 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/CacheMetadata.cs @@ -4,6 +4,7 @@ using System; using System.Data; +using System.Threading.Tasks; using Xunit; namespace Microsoft.Data.SqlClient.ManualTesting.Tests @@ -399,6 +400,73 @@ public static void Test(string dstConstr, string dstTable) } } + public class CacheMetadataAsync + { + private static readonly string sourceTable = "employees"; + private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; + + // Test that CacheMetadata works correctly with WriteToServerAsync. + public static void Test(string srcConstr, string dstConstr, string dstTable) + { + Task t = TestAsync(srcConstr, dstConstr, dstTable); + t.Wait(); + Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); + } + + private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable) + { + string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); + string initialQuery = string.Format(initialQueryTemplate, dstTable); + + using SqlConnection dstConn = new(dstConstr); + using SqlCommand dstCmd = dstConn.CreateCommand(); + dstConn.Open(); + + try + { + Helpers.TryExecute(dstCmd, initialQuery); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable; + + // First WriteToServerAsync: metadata is queried and cached. + using (SqlConnection srcConn = new(srcConstr)) + { + await srcConn.OpenAsync().ConfigureAwait(false); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); + await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 5); + + // Second WriteToServerAsync: should reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) + { + await srcConn.OpenAsync().ConfigureAwait(false); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); + await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 10); + + // Third WriteToServerAsync: should still reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) + { + await srcConn.OpenAsync().ConfigureAwait(false); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); + await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); + } + Helpers.VerifyResults(dstConn, dstTable, 3, 15); + } + finally + { + Helpers.TryExecute(dstCmd, "drop table " + dstTable); + } + } + } + public class CacheMetadataCombinedWithKeepNulls { private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50) default 'DefaultVal', col3 nvarchar(50))"; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs index a93c9f5d47..90a4e5cbf3 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/SqlBulkCopyTest.cs @@ -356,5 +356,11 @@ public void CacheMetadataCombinedWithKeepNullsTest() { CacheMetadataCombinedWithKeepNulls.Test(_connStr, AddGuid("SqlBulkCopyTest_CacheMetadataKeepNulls")); } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] + public void CacheMetadataAsyncTest() + { + CacheMetadataAsync.Test(_connStr, _connStr, AddGuid("SqlBulkCopyTest_CacheMetadataAsync")); + } } } From 58cf9428ce535762360771a66b544819aaeefa45 Mon Sep 17 00:00:00 2001 From: Saransh Sharma Date: Thu, 26 Feb 2026 10:32:46 -0800 Subject: [PATCH 15/15] Improve readability --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 048a90efb6..cbd74f73d6 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -563,9 +563,13 @@ private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result) // Matches associated columns with metadata from initial query. // Builds and executes the update bulk command. - private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults) + // metaDataSet is passed in by the caller so that when CacheMetadata is enabled, the + // caller can supply a clone, allowing this method to null-prune unmatched/rejected + // columns freely without mutating the shared cache. + private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults, _SqlMetaDataSet metaDataSet) { Debug.Assert(internalResults != null, "Where are the results from the initial query?"); + Debug.Assert(metaDataSet != null, "metaDataSet must not be null"); StringBuilder updateBulkCommandText = new StringBuilder(); @@ -609,17 +613,8 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i // the next column in the command text. bool appendComma = false; - // Loop over the metadata for each result column. - // When using cached metadata, clone the metadata set so that null-pruning of - // unmatched/rejected columns does not mutate the shared cache. Without this, - // changing ColumnMappings between WriteToServer calls (e.g. mapping fewer columns - // on the first call, then more on the second) would permanently lose metadata - // entries from the cache. - _SqlMetaDataSet metaDataSet = internalResults[MetaDataResultId].MetaData; - if (CachedMetadata != null) - { - metaDataSet = metaDataSet.Clone(); - } + // Loop over the metadata for each result column, null-pruning unmatched/rejected + // columns. metaDataSet is safe to mutate here — see the call site for clone logic. _operationMetaData = metaDataSet; _sortedColumnMappings = new List<_ColumnMapping>(metaDataSet.Length); for (int i = 0; i < metaDataSet.Length; i++) @@ -2891,7 +2886,14 @@ private void WriteToServerInternalRestContinuedAsync(BulkCopySimpleResultSet int try { - updateBulkCommandText = AnalyzeTargetAndCreateUpdateBulkCommand(internalResults); + // When CacheMetadata is enabled, internalResults IS the cached result set (see + // CreateAndExecuteInitialQueryAsync). Clone the metadata set so that + // AnalyzeTargetAndCreateUpdateBulkCommand can null-prune unmatched/rejected + // columns without mutating the cache across WriteToServer calls. + _SqlMetaDataSet metaDataSet = CachedMetadata != null + ? internalResults[MetaDataResultId].MetaData.Clone() + : internalResults[MetaDataResultId].MetaData; + updateBulkCommandText = AnalyzeTargetAndCreateUpdateBulkCommand(internalResults, metaDataSet); if (_sortedColumnMappings.Count != 0) {