Add Parameterization Validator

Author: YegorStepanov

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -60,6 +60,7 @@ public IEnumerable<IAnalyser> GetAnalysers()
         public IEnumerable<IValidator> GetValidators()
         {
             yield return BaselineValidator.FailOnError;
+            yield return ParameterizationValidator.FailOnError;
             yield return SetupCleanupValidator.FailOnError;
 #if !DEBUG
             yield return JitOptimizationsValidator.FailOnError;
@@ -107,4 +108,4 @@ public string ArtifactsPath
 
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
     }
-}
+}

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -19,6 +19,7 @@ public static class ImmutableConfigBuilder
         private static readonly IValidator[] MandatoryValidators =
         {
             BaselineValidator.FailOnError,
+            ParameterizationValidator.FailOnError,
             SetupCleanupValidator.FailOnError,
             RunModeValidator.FailOnError,
             DiagnosersValidator.Composite,

src/BenchmarkDotNet/Validators/ParameterizationValidator.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Extensions;
+
+namespace BenchmarkDotNet.Validators
+{
+    public class ParameterizationValidator : IValidator
+    {
+        public static readonly ParameterizationValidator FailOnError = new ParameterizationValidator();
+
+        public bool TreatsWarningsAsErrors => true;
+
+        private const BindingFlags ReflectionFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters input) =>
+            ValidateAttributes<ParamsAttribute>(input)
+                .Concat(ValidateAttributes<ParamsSourceAttribute>(input));
+
+        private IEnumerable<ValidationError> ValidateAttributes<T>(ValidationParameters input) where T : Attribute
+        {
+            return input.Benchmarks.Select(benchmark => benchmark.Descriptor.Type)
+                .Distinct()
+                .SelectMany(type => type.GetTypeMembersWithGivenAttribute<T>(ReflectionFlags))
+                .Distinct()
+                .Where(member => !member.IsPublic)
+                .Select(member => GetNonPublicError(member.Name, GetAttributeName(typeof(T))))
+                .Where(error => error != null);
+        }
+
+        private ValidationError GetNonPublicError(string memberName, string attributeName)
+        {
+            return new ValidationError(
+                TreatsWarningsAsErrors,
+                $"Member \"{memberName}\" must be public if it has the [{attributeName}] attribute applied to it");
+        }
+
+        private static string GetAttributeName(Type type) => type.Name.Replace("Attribute", string.Empty);
+    }
+}

tests/BenchmarkDotNet.Tests/Validators/ParameterizationValidatorTests.cs

@@ -0,0 +1,121 @@
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Validators;
+using Xunit;
+
+namespace BenchmarkDotNet.Tests.Validators
+{
+    public class ParameterizationValidatorTests
+    {
+        [Fact]
+        public void PublicIsSupported()
+        {
+            var validationErrors = ParameterizationValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(PublicParameterization)));
+
+            Assert.Empty(validationErrors);
+        }
+
+        public class PublicParameterization
+        {
+#pragma warning disable CS0649
+            [Params(1)]
+            public int Field;
+
+            [Params(1)]
+            public int Property { get; set; }
+
+            [ParamsSource(nameof(GetValues))]
+            public int FieldSource;
+
+            [ParamsSource(nameof(GetValues))]
+            public int PropertySource { get; set; }
+#pragma warning restore CS0649
+
+            public IEnumerable<object> GetValues()
+            {
+                yield return 1;
+            }
+
+            [Benchmark]
+            public void Foo() { }
+        }
+
+        [Fact]
+        public void InternalIsNotSupported()
+        {
+            var validationErrors = ParameterizationValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(InternalParameterization))).ToArray();
+
+            Assert.NotEmpty(validationErrors);
+            Assert.StartsWith("Member \"Field\" must be public if it has the [Params]", validationErrors[0].Message);
+            Assert.StartsWith("Member \"Property\" must be public if it has the [Params]", validationErrors[1].Message);
+            Assert.StartsWith("Member \"FieldSource\" must be public if it has the [ParamsSource]", validationErrors[2].Message);
+            Assert.StartsWith("Member \"PropertySource\" must be public if it has the [ParamsSource]", validationErrors[3].Message);
+        }
+
+        public class InternalParameterization
+        {
+#pragma warning disable CS0649
+            [Params(1)]
+            internal int Field;
+
+            [Params(1)]
+            internal int Property { get; set; }
+
+            [ParamsSource(nameof(GetValues))]
+            internal int FieldSource;
+
+            [ParamsSource(nameof(GetValues))]
+            internal int PropertySource { get; set; }
+#pragma warning restore CS0649
+
+            public IEnumerable<object> GetValues()
+            {
+                yield return 0;
+            }
+
+            [Benchmark]
+            public void Foo() { }
+        }
+
+        [Fact]
+        public void PrivateIsNotSupported()
+        {
+            var validationErrors = ParameterizationValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(PrivateParameterization))).ToArray();
+
+            Assert.NotEmpty(validationErrors);
+            Assert.StartsWith("Member \"Field\" must be public if it has the [Params]", validationErrors[0].Message);
+            Assert.StartsWith("Member \"Property\" must be public if it has the [Params]", validationErrors[1].Message);
+            Assert.StartsWith("Member \"FieldSource\" must be public if it has the [ParamsSource]", validationErrors[2].Message);
+            Assert.StartsWith("Member \"PropertySource\" must be public if it has the [ParamsSource]", validationErrors[3].Message);
+        }
+
+        public class PrivateParameterization
+        {
+#pragma warning disable CA1823
+#pragma warning disable CS0169
+            [Params(1)]
+            private int Field;
+
+            [Params(1)]
+            private int Property { get; set; }
+
+            [ParamsSource(nameof(GetValues))]
+            private int FieldSource;
+
+            [ParamsSource(nameof(GetValues))]
+            private int PropertySource { get; set; }
+#pragma warning restore CA1823
+#pragma warning restore CS0169
+
+            public IEnumerable<object> GetValues()
+            {
+                yield return 0;
+            }
+
+            [Benchmark]
+            public void Foo() { }
+        }
+    }
+}