Feature | Add SqlBulkCopyOptions.CacheMetadata flag

doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml

@@ -235,6 +235,35 @@ This code is provided to demonstrate the syntax for using **SqlBulkCopy** only.
 ]]></format>
       </example>
     </Close>
+    <ClearCachedMetadata>
+      <summary>
+        Clears the cached destination table metadata when using the
+        <see cref="F:Microsoft.Data.SqlClient.SqlBulkCopyOptions.CacheMetadata" />
+        option.
+      </summary>
+      <remarks>
+        <para>
+          Call this method when you know the destination table schema
+          has changed and you want to force the next
+          <c>WriteToServer</c> operation to refresh the metadata
+          from the server.
+        </para>
+        <para>
+          The cache is automatically invalidated when the
+          <see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.DestinationTableName" />
+          property is changed to a different table name.
+        </para>
+        <para>
+          The cache is not automatically invalidated when the
+          connection context changes. Call this method if the
+          underlying
+          <see cref="T:Microsoft.Data.SqlClient.SqlConnection" />
+          changes database (for example, via
+          <see cref="M:System.Data.Common.DbConnection.ChangeDatabase(System.String)" />)
+          or reconnects to a different server due to failover.
+        </para>
+      </remarks>
+    </ClearCachedMetadata>
     <EnableStreaming>
       <summary>
         Enables or disables a <see cref="T:Microsoft.Data.SqlClient.SqlBulkCopy" /> object to stream data from an <see cref="T:System.Data.IDataReader" /> object

doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml

@@ -68,5 +68,46 @@ To see how the option changes the way the bulk load works, run the sample with t
         When specified, each batch of the bulk-copy operation will occur within a transaction. If you indicate this option and also provide a <see cref="T:Microsoft.Data.SqlClient.SqlTransaction" /> object to the constructor, an <see cref="T:System.ArgumentException" /> occurs.
       </summary>
     </UseInternalTransaction>
+    <CacheMetadata>
+      <summary>
+        <para>
+          When specified, <b>CacheMetadata</b> caches destination table
+          metadata after the first bulk copy operation, allowing
+          subsequent operations to the same table to skip the metadata
+          discovery query. This can improve performance when performing
+          multiple bulk copy operations to the same destination table.
+        </para>
+        <para>
+          <b>Warning:</b> Use this option only when you are certain the
+          destination table schema will not change between bulk copy
+          operations. If the table schema changes (columns added,
+          removed, or modified), using cached metadata may result in
+          data corruption, failed operations, or unexpected behavior.
+          Call
+          <see cref="M:Microsoft.Data.SqlClient.SqlBulkCopy.ClearCachedMetadata" />
+          to clear the cache if the schema changes.
+        </para>
+        <para>
+          The cache is automatically invalidated when
+          <see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.DestinationTableName" />
+          is changed to a different table. Changing
+          <see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.ColumnMappings" />
+          between operations does not require cache invalidation
+          because the cached metadata describes only the destination
+          table schema, not the source-to-destination column mapping.
+        </para>
+        <para>
+          The cache is not automatically invalidated when the
+          connection context changes. If the underlying
+          <see cref="T:Microsoft.Data.SqlClient.SqlConnection" />
+          changes database (for example, via
+          <see cref="M:System.Data.Common.DbConnection.ChangeDatabase(System.String)" />)
+          or reconnects to a different server due to failover, callers
+          should call
+          <see cref="M:Microsoft.Data.SqlClient.SqlBulkCopy.ClearCachedMetadata" />
+          to ensure the metadata is refreshed.
+        </para>
+      </summary>
+    </CacheMetadata>
   </members>
 </docs>

src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs

@@ -212,6 +212,8 @@ public SqlBulkCopy(string connectionString, Microsoft.Data.SqlClient.SqlBulkCopy
     public event Microsoft.Data.SqlClient.SqlRowsCopiedEventHandler SqlRowsCopied { add { } remove { } }
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/Close/*'/>
     public void Close() { }
+    /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/ClearCachedMetadata/*'/>
+    public void ClearCachedMetadata() { }
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/System.IDisposable.Dispose/*'/>
     void System.IDisposable.Dispose() { }
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/WriteToServer[@name="DbDataReaderParameter"]/*'/>
@@ -343,6 +345,8 @@ public enum SqlBulkCopyOptions
 {
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/AllowEncryptedValueModifications/*'/>
     AllowEncryptedValueModifications = 64,
+    /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CacheMetadata/*'/>
+    CacheMetadata = 128,
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CheckConstraints/*'/>
     CheckConstraints = 2,
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/Default/*'/>

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs

@@ -235,6 +235,13 @@ private int RowNumber
 
         private SourceColumnMetadata[] _currentRowMetadata;
 
+        // Metadata caching fields for CacheMetadata option
+        internal BulkCopySimpleResultSet CachedMetadata { get; private set; }
+        // Per-operation clone of the destination table metadata, used when CacheMetadata is
+        // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not
+        // mutate the cached BulkCopySimpleResultSet.
+        private _SqlMetaDataSet _operationMetaData;
+
 #if DEBUG
         internal static bool s_setAlwaysTaskOnWrite; //when set and in DEBUG mode, TdsParser::WriteBulkCopyValue will always return a task
         internal static bool SetAlwaysTaskOnWrite
@@ -353,6 +360,12 @@ public string DestinationTableName
                 {
                     throw ADP.ArgumentOutOfRange(nameof(DestinationTableName));
                 }
+                else if (string.Equals(_destinationTableName, value, StringComparison.Ordinal))
+                {
+                    return;
+                }
+
+                CachedMetadata = null;
                 _destinationTableName = value;
             }
         }
@@ -497,6 +510,14 @@ IF EXISTS (SELECT TOP 1 * FROM sys.all_columns WHERE [object_id] = OBJECT_ID('sy
         // We need to have a _parser.RunAsync to make it real async.
         private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result)
         {
+            // Check if we have valid cached metadata for the current destination table
+            if (CachedMetadata != null)
+            {
+                SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName);
+                result = CachedMetadata;
+                return null;
+            }
+
             string TDSCommand = CreateInitialQuery();
             SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Initial Query: '{0}'", TDSCommand);
             SqlClientEventSource.Log.TryCorrelationTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Correlation | Object Id {0}, Activity Id {1}", ObjectID, ActivityCorrelator.Current);
@@ -506,6 +527,7 @@ private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out Bulk
             {
                 result = new BulkCopySimpleResultSet();
                 RunParser(result);
+                CacheMetadataIfEnabled(result);
                 return null;
             }
             else
@@ -523,17 +545,31 @@ private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out Bulk
                     {
                         var internalResult = new BulkCopySimpleResultSet();
                         RunParserReliably(internalResult);
+                        CacheMetadataIfEnabled(internalResult);
                         return internalResult;
                     }
                 }, TaskScheduler.Default);
             }
         }
 
+        private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result)
+        {
+            if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata))
+            {
+                CachedMetadata = result;
+                SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CacheMetadataIfEnabled | Info | Cached metadata for table '{0}'", _destinationTableName);
+            }
+        }
+
         // Matches associated columns with metadata from initial query.
         // Builds and executes the update bulk command.
-        private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults)
+        // metaDataSet is passed in by the caller so that when CacheMetadata is enabled, the
+        // caller can supply a clone, allowing this method to null-prune unmatched/rejected
+        // columns freely without mutating the shared cache.
+        private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults, _SqlMetaDataSet metaDataSet)
         {
             Debug.Assert(internalResults != null, "Where are the results from the initial query?");
+            Debug.Assert(metaDataSet != null, "metaDataSet must not be null");
 
             StringBuilder updateBulkCommandText = new StringBuilder();
 
@@ -577,8 +613,9 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i
             // the next column in the command text.
             bool appendComma = false;
 
-            // Loop over the metadata for each result column.
-            _SqlMetaDataSet metaDataSet = internalResults[MetaDataResultId].MetaData;
+            // Loop over the metadata for each result column, null-pruning unmatched/rejected
+            // columns. metaDataSet is safe to mutate here — see the call site for clone logic.
+            _operationMetaData = metaDataSet;
             _sortedColumnMappings = new List<_ColumnMapping>(metaDataSet.Length);
             for (int i = 0; i < metaDataSet.Length; i++)
             {
@@ -875,11 +912,18 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults)
         {
             _stateObj.SetTimeoutSeconds(BulkCopyTimeout);
 
-            _SqlMetaDataSet metadataCollection = internalResults[MetaDataResultId].MetaData;
+            _SqlMetaDataSet metadataCollection = _operationMetaData ?? internalResults[MetaDataResultId].MetaData;
             _stateObj._outputMessageType = TdsEnums.MT_BULK;
             _parser.WriteBulkCopyMetaData(metadataCollection, _sortedColumnMappings.Count, _stateObj);
         }
 
+        /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/ClearCachedMetadata/*'/>
+        public void ClearCachedMetadata()
+        {
+            CachedMetadata = null;
+            SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.ClearCachedMetadata | Info | Metadata cache cleared");
+        }
+
         // Terminates the bulk copy operation.
         // Must be called at the end of the bulk copy session.
         /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/Close/*'/>
@@ -900,6 +944,8 @@ private void Dispose(bool disposing)
                 // Dispose dependent objects
                 _columnMappings = null;
                 _parser = null;
+                CachedMetadata = null;
+                _operationMetaData = null;
                 try
                 {
                     // Just in case there is a lingering transaction (which there shouldn't be)
@@ -2667,7 +2713,7 @@ private Task CopyBatchesAsyncContinued(BulkCopySimpleResultSet internalResults,
 
                 // Load encryption keys now (if needed)
                 _parser.LoadColumnEncryptionKeys(
-                    internalResults[MetaDataResultId].MetaData,
+                    _operationMetaData ?? internalResults[MetaDataResultId].MetaData,
                     _connection);
 
                 Task task = CopyRowsAsync(0, _savedBatchSize, cts); // This is copying 1 batch of rows and setting _hasMoreRowToCopy = true/false.
@@ -2840,7 +2886,14 @@ private void WriteToServerInternalRestContinuedAsync(BulkCopySimpleResultSet int
 
             try
             {
-                updateBulkCommandText = AnalyzeTargetAndCreateUpdateBulkCommand(internalResults);
+                // When CacheMetadata is enabled, internalResults IS the cached result set (see
+                // CreateAndExecuteInitialQueryAsync). Clone the metadata set so that
+                // AnalyzeTargetAndCreateUpdateBulkCommand can null-prune unmatched/rejected
+                // columns without mutating the cache across WriteToServer calls.
+                _SqlMetaDataSet metaDataSet = CachedMetadata != null
+                    ? internalResults[MetaDataResultId].MetaData.Clone()
+                    : internalResults[MetaDataResultId].MetaData;
+                updateBulkCommandText = AnalyzeTargetAndCreateUpdateBulkCommand(internalResults, metaDataSet);
 
                 if (_sortedColumnMappings.Count != 0)
                 {
@@ -3194,6 +3247,7 @@ private void ResetWriteToServerGlobalVariables()
             _dataTableSource = null;
             _dbDataReaderRowSource = null;
             _isAsyncBulkCopy = false;
+            _operationMetaData = null;
             _rowEnumerator = null;
             _rowSource = null;
             _rowSourceType = ValueSourceType.Unspecified;

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopyOptions.cs

@@ -33,6 +33,9 @@ public enum SqlBulkCopyOptions
 
         /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/AllowEncryptedValueModifications/*'/>
         AllowEncryptedValueModifications = 1 << 6,
+
+        /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CacheMetadata/*'/>
+        CacheMetadata = 1 << 7,
     }
 }
 

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs

@@ -352,6 +352,7 @@ private _SqlMetaDataSet(_SqlMetaDataSet original)
             _visibleColumnMap = original._visibleColumnMap;
             dbColumnSchema = original.dbColumnSchema;
             schemaTable = original.schemaTable;
+            cekTable = original.cekTable;
 
             if (original._metaDataArray == null)
             {
@@ -577,6 +578,10 @@ internal virtual void CopyFrom(SqlMetaDataPriv original)
                 xmlSchemaCollection = new SqlMetaDataXmlSchemaCollection();
                 xmlSchemaCollection.CopyFrom(original.xmlSchemaCollection);
             }
+
+            this.isEncrypted = original.isEncrypted;
+            this.baseTI = original.baseTI;
+            this.cipherMD = original.cipherMD;
         }
     }
 

src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj

@@ -111,6 +111,7 @@
     <Compile Include="SQL\SqlBulkCopyTest\Bug85007.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\Bug903514.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\Bug98182.cs" />
+    <Compile Include="SQL\SqlBulkCopyTest\CacheMetadata.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\CheckConstraints.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\CopyWidenNullInexactNumerics.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\DataConversionErrorMessageTest.cs" />