From 5480286579da01146e30e80937472897c104435f Mon Sep 17 00:00:00 2001 From: Gerald Schermann Date: Tue, 6 Oct 2026 14:54:21 +0200 Subject: [PATCH] AVRO-4357: [C#] Resolve schema types without type scans ObjectCreator.FindType resolved every schema name that is not in the entry assembly by enumerating all types of all loaded assemblies. In large applications this takes seconds and gigabytes of allocations on the first deserialization of a schema. Look up names with a namespace directly in each loaded assembly and keep the enumeration as fallback. The resolved types stay the same, including the last match winning across assemblies. --- .../benchmark/ObjectCreatorBenchmarks.cs | 118 ++++++++++++++++++ .../src/apache/main/Specific/ObjectCreator.cs | 97 ++++++++++---- .../test/Specific/ObjectCreatorTests.cs | 45 +++++++ 3 files changed, 238 insertions(+), 22 deletions(-) create mode 100644 lang/csharp/src/apache/benchmark/ObjectCreatorBenchmarks.cs diff --git a/lang/csharp/src/apache/benchmark/ObjectCreatorBenchmarks.cs b/lang/csharp/src/apache/benchmark/ObjectCreatorBenchmarks.cs new file mode 100644 index 00000000000..656b0865cdf --- /dev/null +++ b/lang/csharp/src/apache/benchmark/ObjectCreatorBenchmarks.cs @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using Avro.Specific; +using BenchmarkDotNet.Attributes; + +namespace Avro.Benchmark +{ + /// + /// Measures how long takes to resolve the generated classes of a schema + /// the first time, depending on how many types the application has loaded. A fresh + /// per operation keeps its type cache cold, which is what every + /// process pays once for every named schema type it deserializes. + /// + [SimpleJob(launchCount: 1, warmupCount: 2, iterationCount: 10)] + [MemoryDiagnoser] + public class ObjectCreatorBenchmarks + { + private const int NamedSchemaTypes = 50; + private const int TypesPerApplicationAssembly = 1_000; + + private string[] _namedSchemaTypeNames; + + /// + /// Number of types loaded in addition to the ones of the benchmark process, spread over + /// assemblies of 1,000 types each. An ASP.NET Core application loads roughly 25,000 types, + /// large applications several 100,000. + /// + [Params(0, 25_000, 100_000, 250_000)] + public int AdditionalLoadedTypes { get; set; } + + [GlobalSetup] + public void Setup() + { + for (int i = 0; i < AdditionalLoadedTypes / TypesPerApplicationAssembly; i++) + { + DefineAssembly($"Bench.Application{i}", TypesPerApplicationAssembly); + } + + // The generated classes live in their own assembly, like generated contracts referenced by a service + _namedSchemaTypeNames = DefineAssembly("Bench.Contracts.V1", NamedSchemaTypes) + .Select(type => type.FullName) + .ToArray(); + + Console.WriteLine($"// Loaded assemblies: {AppDomain.CurrentDomain.GetAssemblies().Length}, types: {CountLoadedTypes()}"); + } + + [Benchmark] + public void ResolveNamedSchemaTypes() + { + ObjectCreator objectCreator = new ObjectCreator(); + foreach (string name in _namedSchemaTypeNames) + { + objectCreator.GetType(name, Schema.Type.Record); + } + } + + private static IEnumerable DefineAssembly(string assemblyName, int typeCount) + { + AssemblyBuilder assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly( + new AssemblyName(assemblyName), + AssemblyBuilderAccess.Run + ); + ModuleBuilder moduleBuilder = assemblyBuilder.DefineDynamicModule(assemblyName); + List types = new List(typeCount); + for (int i = 0; i < typeCount; i++) + { + types.Add( + moduleBuilder + .DefineType( + $"{assemblyName}.Type{i}", + TypeAttributes.Public | TypeAttributes.Class + ) + .CreateType() + ); + } + + return types; + } + + private static int CountLoadedTypes() + { + int count = 0; + foreach (Assembly assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + try + { + count += assembly.GetTypes().Length; + } + catch (ReflectionTypeLoadException e) + { + count += e.Types.Count(type => type != null); + } + } + + return count; + } + } +} diff --git a/lang/csharp/src/apache/main/Specific/ObjectCreator.cs b/lang/csharp/src/apache/main/Specific/ObjectCreator.cs index 073b107958a..60225f820a1 100644 --- a/lang/csharp/src/apache/main/Specific/ObjectCreator.cs +++ b/lang/csharp/src/apache/main/Specific/ObjectCreator.cs @@ -107,30 +107,10 @@ private Type FindType(string name) type = Type.GetType(name); } - // type is still not found, need to loop through all loaded assemblies + // type is still not found, need to look through all loaded assemblies if (type == null) { - foreach (Assembly assembly in AppDomain.CurrentDomain.GetAssemblies()) - { - // Loading all types from all assemblies could fail for a variety of - // non -fatal reasons. If we fail to load types from an assembly, continue. - try - { - // Change the search to look for Types by both NAME and FULLNAME - foreach (Type t in assembly.GetTypes()) - { - if (name == t.Name || name == t.FullName || CodeGenUtil.Instance.UnMangle(name) == t.FullName) - { - type = t; - break; - } - } - } - catch - { - continue; - } - } + type = FindTypeInLoadedAssemblies(name); } return type @@ -288,6 +268,79 @@ public Type GetType(Schema schema) return FindType(schema.Name); } + /// + /// Finds the type with the given name in all loaded assemblies. If several assemblies contain + /// a matching type, the one found last in the loaded assemblies wins. + /// + /// the object type to locate + /// the object type, or null if not found + private static Type FindTypeInLoadedAssemblies(string name) + { + Assembly[] assemblies = AppDomain.CurrentDomain.GetAssemblies(); + string unmangledName = CodeGenUtil.Instance.UnMangle(name); + + // A name with a namespace can only match the full name of a type. Looking it up directly + // in each assembly finds the same type as enumerating all types of all assemblies, without + // loading all of them. + if (name.IndexOf('.') >= 0) + { + Type type = null; + foreach (Assembly assembly in assemblies) + { + try + { + type = assembly.GetType(name, false) ?? assembly.GetType(unmangledName, false) ?? type; + } + catch + { + // Looking up a type could fail for a variety of non-fatal reasons, continue. + } + } + + if (type != null) + { + return type; + } + } + + return FindTypeByEnumeratingLoadedAssemblies(assemblies, name, unmangledName); + } + + /// + /// Finds the type with the given name or full name by enumerating all types of all loaded assemblies. + /// + /// the loaded assemblies + /// the object type to locate + /// the object type to locate, without escaped reserved words + /// the object type, or null if not found + private static Type FindTypeByEnumeratingLoadedAssemblies(Assembly[] assemblies, string name, string unmangledName) + { + Type type = null; + foreach (Assembly assembly in assemblies) + { + // Loading all types from all assemblies could fail for a variety of + // non -fatal reasons. If we fail to load types from an assembly, continue. + try + { + // Change the search to look for Types by both NAME and FULLNAME + foreach (Type t in assembly.GetTypes()) + { + if (name == t.Name || name == t.FullName || unmangledName == t.FullName) + { + type = t; + break; + } + } + } + catch + { + continue; + } + } + + return type; + } + /// /// Gets the type of the specified type name /// diff --git a/lang/csharp/src/apache/test/Specific/ObjectCreatorTests.cs b/lang/csharp/src/apache/test/Specific/ObjectCreatorTests.cs index d6b29d5d954..33099acc09a 100644 --- a/lang/csharp/src/apache/test/Specific/ObjectCreatorTests.cs +++ b/lang/csharp/src/apache/test/Specific/ObjectCreatorTests.cs @@ -20,6 +20,9 @@ using NUnit.Framework; using System; using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; namespace Avro.Test.Specific { @@ -103,6 +106,48 @@ public void TestComplexGetTypes(Type expecteType, string name) Assert.AreEqual(expecteType, objectCreator.GetType(name, Schema.Type.Record)); } + [Test] + public void TestGetTypeByFullNameFromLoadedAssembly() + { + var name = $"Avro.Test.Dynamic{Guid.NewGuid():N}.Contracts.MyRecord"; + var expected = DefineAssemblyWithType(name).GetType(name); + var objectCreator = new ObjectCreator(); + + Assert.AreEqual(expected, objectCreator.GetType(name, Schema.Type.Record)); + } + + [TestCase("Avro.Test.Specific.return.Record", TestName = "TestGetTypeReservedWordInNamespace_Unmangled")] + [TestCase("Avro.Test.Specific.@return.Record", TestName = "TestGetTypeReservedWordInNamespace_Mangled")] + public void TestGetTypeReservedWordInNamespace(string name) + { + var objectCreator = new ObjectCreator(); + + Assert.AreEqual(typeof(Avro.Test.Specific.@return.Record), objectCreator.GetType(name, Schema.Type.Record)); + } + + [Test] + public void TestGetTypeFullNameInSeveralAssembliesResolvesToLastLoadedAssembly() + { + var name = $"Avro.Test.Dynamic{Guid.NewGuid():N}.Contracts.MyRecord"; + var first = DefineAssemblyWithType(name); + var second = DefineAssemblyWithType(name); + var lastInLoadOrder = AppDomain.CurrentDomain.GetAssemblies() + .Last(assembly => assembly.FullName == first.FullName || assembly.FullName == second.FullName); + var objectCreator = new ObjectCreator(); + + Assert.AreEqual(lastInLoadOrder.GetType(name), objectCreator.GetType(name, Schema.Type.Record)); + Assert.AreNotEqual(first.GetType(name), second.GetType(name)); + } + + private static Assembly DefineAssemblyWithType(string typeFullName) + { + var assemblyName = $"Avro.Test.Dynamic{Guid.NewGuid():N}"; + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(assemblyName), AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule(assemblyName); + moduleBuilder.DefineType(typeFullName, TypeAttributes.Public | TypeAttributes.Class).CreateType(); + return assemblyBuilder; + } + private class MyNullableFoo { }