unity/Assets/Game/Scripts/LightingValidation.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Text;
using UnityEngine;
using UnityEngine.InputSystem;
using UnityEngine.InputSystem.LowLevel;
using UnityEngine.Profiling;
using UnityEngine.Rendering;

namespace Ironspore
{
    // Opt-in player QA. Normal launches never instantiate this component.
    public sealed class LightingValidation : MonoBehaviour
    {
        const BindingFlags Private = BindingFlags.Instance | BindingFlags.NonPublic;
        static readonly FieldInfo Yaw = typeof(MechPilot).GetField("yaw", Private);
        static readonly FieldInfo Pitch = typeof(MechPilot).GetField("pitch", Private);
        static readonly PropertyInfo View = typeof(MechPilot).GetProperty("FirstPerson");
        static readonly Vector3[] Route = { new Vector3(3, 0, -18), new Vector3(26, 0, -18), new Vector3(26, 0, 38), new Vector3(3, 0, 38), new Vector3(3, 0, -18), new Vector3(-26, 0, -18), new Vector3(-26, 0, 38), new Vector3(3, 0, 38) };
        readonly FrameTiming[] timing = new FrameTiming[1];
        readonly List<Sample> samples = new List<Sample>(40000);
        readonly Report report = new Report();
        GameSession game;
        Keyboard keyboard;
        Mouse mouse;
        Camera benchmarkCamera;
        int renderedFrames;
        string folder;
        bool measuring;
        double previous;
        ulong lastTiming;
        int waypoint;
        int visitedWaypoints, segmentWaypoints;
        float travelled, segmentTravelled;
        bool touring;
        Color32[] captured;

        [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.AfterSceneLoad)]
        static void Launch()
        {
            if (!Debug.isDebugBuild || Array.IndexOf(Environment.GetCommandLineArgs(), "-lighting-qa") < 0)
                return;
            new GameObject("LightingValidation").AddComponent<LightingValidation>();
        }

        static string Arg(string name, string fallback)
        {
            var args = Environment.GetCommandLineArgs();
            int index = Array.IndexOf(args, name);
            return index >= 0 && index + 1 < args.Length ? args[index + 1] : fallback;
        }

        IEnumerator Start()
        {
            folder = Path.GetFullPath(Arg("-lighting-output", Path.Combine(Application.persistentDataPath, "LightingQA")));
            Directory.CreateDirectory(folder);
            Directory.CreateDirectory(Path.Combine(folder, "Screenshots"));
            string isolated = Arg("-ironspore-data-path", "");
            if (!Path.IsPathRooted(isolated) || !string.Equals(GameDataPaths.DirectoryPath, Path.GetFullPath(isolated), StringComparison.OrdinalIgnoreCase))
                throw new InvalidOperationException("Lighting QA requires an absolute -ironspore-data-path.");
            Application.runInBackground = true;
            Screen.SetResolution(1920, 1080, FullScreenMode.FullScreenWindow);
            InputSystem.settings.backgroundBehavior = InputSettings.BackgroundBehavior.IgnoreFocus;
            while (!GameSession.Active)
                yield return null;
            game = GameSession.Active;
            Camera.onPostRender += CameraRendered;
            yield return null;
            string quality = Arg("-lighting-quality", "Medium");
            int level = Array.IndexOf(QualitySettings.names, quality);
            if (level < 0)
                throw new ArgumentException("Unknown quality: " + quality);
            QualitySettings.SetQualityLevel(level, true);
            game.World.BuildHangar();
            QualitySettings.vSyncCount = 0;
            Application.targetFrameRate = int.Parse(Arg("-lighting-cap", "-1"), CultureInfo.InvariantCulture);
            report.tag = Arg("-lighting-tag", "optimized");
            report.branch = int.Parse(Arg("-lighting-branch", "0"), CultureInfo.InvariantCulture);
            report.routeVersion = "perimeter-and-center-v2";
            report.startedUtc = DateTime.UtcNow.ToString("O");
            report.gpu = SystemInfo.graphicsDeviceName;
            report.api = SystemInfo.graphicsDeviceType.ToString();
            report.cpu = SystemInfo.processorType;
            report.gpuMemoryMiB = SystemInfo.graphicsMemorySize;
            report.systemMemoryMiB = SystemInfo.systemMemorySize;
            report.colorSpace = QualitySettings.activeColorSpace.ToString();
            report.quality = QualitySettings.names[level];
            report.shadowDistance = QualitySettings.shadowDistance;
            report.cascades = QualitySettings.shadowCascades;
            report.vSync = QualitySettings.vSyncCount;
            report.frameCap = Application.targetFrameRate;
            report.persistentDataPath = Application.persistentDataPath;
            report.dataDirectory = GameDataPaths.DirectoryPath;
            report.frameTimingEnabled = FrameTimingManager.IsFeatureEnabled();
            keyboard = InputSystem.AddDevice<Keyboard>("LightingQaKeyboard");
            mouse = InputSystem.AddDevice<Mouse>("LightingQaMouse");
            game.Input.devices = new InputDevice[] { keyboard, mouse };
            yield return new WaitForSecondsRealtime(2);
            yield return CaptureScreen("hangar");
            game.StartTrial();
            yield return new WaitForSecondsRealtime(2);
            report.dynamicResolution = game.Pilot.GetComponentInChildren<Camera>().allowDynamicResolution;
            report.renderWidthScale = ScalableBufferManager.widthScaleFactor;
            report.renderHeightScale = ScalableBufferManager.heightScaleFactor;
            report.sunResolution = GameObject.Find("Sun").GetComponent<Light>().shadowCustomResolution;
            yield return VisualChecks();
            if (Array.IndexOf(Environment.GetCommandLineArgs(), "-lighting-visual-only") < 0)
            {
                // Use the actual combat world, including the final-stage enemy population.
                game.EndSortie(true);
                game.BeginSortie();
                game.Store.Commit(save => { save.run.seed = 1739; save.run.stage = 4; save.run.branch = report.branch; save.run.armor = 100000; return true; });
                game.Resume();
                benchmarkCamera = game.Pilot.GetComponentInChildren<Camera>();
                report.cameraWidth = benchmarkCamera.pixelWidth;
                report.cameraHeight = benchmarkCamera.pixelHeight;
                yield return new WaitForSecondsRealtime(2);
                foreach (var enemy in FindObjectsByType<EnemyUnit>())
                    // Keep the same combatants alive throughout every performance sample.
                    typeof(EnemyUnit).GetField("health", Private).SetValue(enemy, 1000000f);
                game.World.CallExtraction();
                touring = true;
                waypoint = 0;
                double warmup = double.Parse(Arg("-lighting-warmup", "60"), CultureInfo.InvariantCulture);
                double duration = double.Parse(Arg("-lighting-seconds", "120"), CultureInfo.InvariantCulture);
                int repeats = int.Parse(Arg("-lighting-repeats", "3"), CultureInfo.InvariantCulture);
                report.warmupSeconds = warmup;
                report.sampleSeconds = duration;
                Debug.Log("Lighting QA warmup: " + warmup + " seconds");
                yield return Wait(warmup);
                for (int repeat = 0; repeat < repeats; repeat++)
                {
                    for (int view = 0; view < 2; view++)
                    {
                        View.SetValue(game.Pilot, view == 1);
                        yield return Wait(2);
                        samples.Clear();
                        renderedFrames = 0;
                        segmentWaypoints = visitedWaypoints;
                        segmentTravelled = travelled;
                        previous = Time.realtimeSinceStartupAsDouble;
                        measuring = true;
                        yield return Wait(duration);
                        measuring = false;
                        string name = (view == 0 ? "shoulder" : "cockpit") + "-" + (repeat + 1);
                        WriteSegment(name);
                        yield return CaptureScreen(name);
                        Debug.Log("Lighting QA completed " + name);
                    }
                }
                touring = false;
            }
            report.width = Screen.width;
            report.height = Screen.height;
            report.finishedUtc = DateTime.UtcNow.ToString("O");
            report.completed = true;
            report.routeCompleted = game.InField && game.Run.routeCompleted;
            SaveReport();
            if (game.InField) game.EndSortie(true);
            Debug.Log("Lighting QA finished: " + Path.Combine(folder, "report.json"));
            if (!Application.isEditor)
                Application.Quit();
        }

        static IEnumerator Wait(double seconds)
        {
            double until = Time.realtimeSinceStartupAsDouble + seconds;
            while (Time.realtimeSinceStartupAsDouble < until)
                yield return null;
        }

        IEnumerator VisualChecks()
        {
            var pilot = game.Pilot;
            var visual = pilot.transform.Find("MechVisual");
            var renderers = visual.GetComponentsInChildren<Renderer>(true);
            var sun = GameObject.Find("Sun").GetComponent<Light>();
            Pitch.SetValue(pilot, 25f);
            Yaw.SetValue(pilot, 0f);
            View.SetValue(pilot, false);
            yield return new WaitForSecondsRealtime(.25f);
            yield return CaptureScreen("shoulder");
            View.SetValue(pilot, true);
            yield return new WaitForSecondsRealtime(.25f);
            yield return CaptureScreen("cockpit");
            int activeCasters = 0;
            foreach (var renderer in renderers)
                if (renderer.enabled && renderer.gameObject.activeInHierarchy && renderer.shadowCastingMode == ShadowCastingMode.ShadowsOnly)
                    activeCasters++;
            report.firstPersonShadowOnlyRenderers = activeCasters;
            // Hold the real pilot in first-person mode. The observer sees its shadow only.
            pilot.enabled = false;
            var cockpit = (GameObject)typeof(MechPilot).GetField("cockpit", Private).GetValue(pilot);
            cockpit.SetActive(false);
            var observerObject = new GameObject("LightingQaObserver");
            var observer = observerObject.AddComponent<Camera>();
            observer.enabled = false;
            observer.cullingMask = ~(1 << 11);
            observer.fieldOfView = 52;
            observer.nearClipPlane = .1f;
            observer.farClipPlane = 140;
            observer.transform.position = new Vector3(0, 9, -23);
            observer.transform.LookAt(new Vector3(0, 0, -15));
            yield return null;
            CaptureCamera(observer, "shadow-before-move");
            var before = captured;
            var controller = pilot.GetComponent<CharacterController>();
            controller.enabled = false;
            pilot.transform.position += Vector3.right * 2;
            controller.enabled = true;
            yield return null;
            CaptureCamera(observer, "shadow-after-move");
            report.movingShadowPixels = DifferentPixels(before, captured);
            var oldShadows = sun.shadows;
            sun.shadows = LightShadows.None;
            yield return null;
            CaptureCamera(observer, "sun-shadows-disabled");
            var noShadows = captured;
            sun.shadows = oldShadows;
            yield return null;
            CaptureCamera(observer, "sun-shadows-enabled");
            report.sunShadowPixels = DifferentPixels(noShadows, captured);
            var localObject = GameObject.Find("NavigationWorkLight");
            if (localObject)
            {
                var local = localObject.GetComponent<Light>();
                controller.enabled = false;
                pilot.transform.position = new Vector3(0, .1f, 18);
                controller.enabled = true;
                observer.transform.position = new Vector3(7, 10, 13);
                observer.transform.LookAt(new Vector3(0, .3f, 21));
                yield return new WaitForSecondsRealtime(.25f);
                local.enabled = false;
                var manager = local.GetComponentInParent<MonoBehaviour>();
                if (manager && manager.GetType().Name == "WorldLighting")
                    manager.enabled = false;
                CaptureCamera(observer, "local-off");
                var off = captured;
                local.enabled = true;
                CaptureCamera(observer, "local-on");
                var on = captured;
                var blocker = GameObject.CreatePrimitive(PrimitiveType.Cube);
                blocker.name = "LightingQaTemporaryBlocker";
                blocker.transform.position = local.transform.position + local.transform.forward * 1.1f;
                blocker.transform.localScale = new Vector3(1.4f, .15f, 1.4f);
                blocker.GetComponent<Collider>().enabled = false;
                local.enabled = false;
                yield return null;
                CaptureCamera(observer, "local-blocked-off");
                var blockedOff = captured;
                local.enabled = true;
                CaptureCamera(observer, "local-blocked-on");
                int lit = 0, blocked = 0;
                for (int i = 0; i < on.Length; i++)
                {
                    int direct = Brightness(on[i]) - Brightness(off[i]);
                    int obstructed = Brightness(captured[i]) - Brightness(blockedOff[i]);
                    if (direct > 30) { lit++; if (obstructed < direct * .45f) blocked++; }
                }
                report.localLitPixels = lit;
                report.localBlockedPixels = blocked;
                report.localLightCastsShadows = local.shadows != LightShadows.None;
                Destroy(blocker);
                if (manager) manager.enabled = true;
            }
            Destroy(observerObject);
            pilot.enabled = true;
            report.visualPassed = activeCasters > 0 && report.movingShadowPixels > 50 && report.sunShadowPixels > 50 && report.localLightCastsShadows && report.localBlockedPixels > 50;
            SaveReport();
        }

        static int Brightness(Color32 color) => color.r + color.g + color.b;
        static int DifferentPixels(Color32[] first, Color32[] second)
        {
            int changed = 0;
            for (int i = 0; i < first.Length; i++)
                if (Math.Abs(first[i].r - second[i].r) + Math.Abs(first[i].g - second[i].g) + Math.Abs(first[i].b - second[i].b) > 30)
                    changed++;
            return changed;
        }

        void CaptureCamera(Camera camera, string name)
        {
            var target = RenderTexture.GetTemporary(1920, 1080, 24, RenderTextureFormat.ARGB32);
            var oldTarget = camera.targetTexture;
            var oldActive = RenderTexture.active;
            camera.targetTexture = target;
            camera.Render();
            RenderTexture.active = target;
            var image = new Texture2D(1920, 1080, TextureFormat.RGB24, false);
            image.ReadPixels(new Rect(0, 0, 1920, 1080), 0, 0);
            image.Apply();
            captured = image.GetPixels32();
            File.WriteAllBytes(Path.Combine(folder, "Screenshots", name + ".png"), image.EncodeToPNG());
            camera.targetTexture = oldTarget;
            RenderTexture.active = oldActive;
            RenderTexture.ReleaseTemporary(target);
            Destroy(image);
        }

        IEnumerator CaptureScreen(string name)
        {
            yield return new WaitForEndOfFrame();
            var image = ScreenCapture.CaptureScreenshotAsTexture();
            File.WriteAllBytes(Path.Combine(folder, "Screenshots", name + ".png"), image.EncodeToPNG());
            Destroy(image);
        }

        void Update()
        {
            if (!touring || !game || !game.InField)
                return;
            var pilot = game.Pilot;
            var goal = Route[waypoint];
            var offset = goal - pilot.transform.position;
            offset.y = 0;
            if (offset.magnitude < 1.2f) { visitedWaypoints++; waypoint = (waypoint + 1) % Route.Length; offset = Route[waypoint] - pilot.transform.position; offset.y = 0; }
            Yaw.SetValue(pilot, Mathf.Atan2(offset.x, offset.z) * Mathf.Rad2Deg);
            Pitch.SetValue(pilot, 12f);
            var beforeMove = pilot.transform.position;
            pilot.GetComponent<CharacterController>().Move(offset.normalized * (3.5f * Time.deltaTime));
            travelled += Vector3.Distance(beforeMove, pilot.transform.position);
            var keys = new KeyboardState();
            if ((int)Time.realtimeSinceStartup % 9 == 0) keys.Press(Key.Space);
            InputSystem.QueueStateEvent(keyboard, keys);
            var buttons = new MouseState();
            if ((int)Time.realtimeSinceStartup % 3 == 0) buttons = buttons.WithButton(MouseButton.Left);
            InputSystem.QueueStateEvent(mouse, buttons);
        }

        void LateUpdate()
        {
            FrameTimingManager.CaptureFrameTimings();
            if (!measuring)
                return;
            double now = Time.realtimeSinceStartupAsDouble;
            var sample = new Sample { wallMs = (now - previous) * 1000 };
            previous = now;
            if (FrameTimingManager.GetLatestTimings(1, timing) > 0 && timing[0].frameStartTimestamp != lastTiming)
            {
                lastTiming = timing[0].frameStartTimestamp;
                sample.cpuMs = timing[0].cpuFrameTime;
                sample.gpuMs = timing[0].gpuFrameTime;
                sample.mainMs = timing[0].cpuMainThreadFrameTime;
                sample.renderMs = timing[0].cpuRenderThreadFrameTime;
                sample.presentWaitMs = timing[0].cpuMainThreadPresentWaitTime;
            }
            samples.Add(sample);
        }

        void CameraRendered(Camera camera)
        {
            if (measuring && camera == benchmarkCamera) renderedFrames++;
        }

        void WriteSegment(string name)
        {
            var wall = new List<double>(samples.Count);
            var gpu = new List<double>(samples.Count);
            var cpu = new List<double>(samples.Count);
            var csv = new StringBuilder("wallMs,cpuMs,gpuMs,mainMs,renderMs,presentWaitMs\n");
            double total = 0;
            foreach (var s in samples)
            {
                wall.Add(s.wallMs); total += s.wallMs;
                if (s.gpuMs > 0) gpu.Add(s.gpuMs);
                if (s.cpuMs > 0) cpu.Add(s.cpuMs);
                csv.AppendFormat(CultureInfo.InvariantCulture, "{0:F5},{1:F5},{2:F5},{3:F5},{4:F5},{5:F5}\n", s.wallMs, s.cpuMs, s.gpuMs, s.mainMs, s.renderMs, s.presentWaitMs);
            }
            wall.Sort(); gpu.Sort(); cpu.Sort();
            File.WriteAllText(Path.Combine(folder, name + ".csv"), csv.ToString());
            report.segments.Add(new Segment { name = name, frames = samples.Count, cameraRenderFrames = renderedFrames, meanFps = samples.Count * 1000.0 / total, p50Ms = Percentile(wall, .5), p95Ms = Percentile(wall, .95), p99Ms = Percentile(wall, .99), gpuP95Ms = Percentile(gpu, .95), cpuP95Ms = Percentile(cpu, .95), gpuSamples = gpu.Count, cpuSamples = cpu.Count, allocatedMiB = Profiler.GetTotalAllocatedMemoryLong() / 1048576.0, enemyCount = FindObjectsByType<EnemyUnit>().Length, width = Screen.width, height = Screen.height, waypointsVisited = visitedWaypoints - segmentWaypoints, distanceTravelled = travelled - segmentTravelled, passed = Screen.width == 1920 && Screen.height == 1080 && Percentile(wall, .95) <= 16.7 && visitedWaypoints - segmentWaypoints >= 4 && renderedFrames >= samples.Count * .98 });
            SaveReport();
        }

        static double Percentile(List<double> sorted, double fraction) => sorted.Count == 0 ? 0 : sorted[Math.Max(0, (int)Math.Ceiling(sorted.Count * fraction) - 1)];
        void SaveReport() => File.WriteAllText(Path.Combine(folder, "report.json"), JsonUtility.ToJson(report, true));

        void OnDestroy()
        {
            Camera.onPostRender -= CameraRendered;
            if (keyboard != null && keyboard.added) InputSystem.RemoveDevice(keyboard);
            if (mouse != null && mouse.added) InputSystem.RemoveDevice(mouse);
        }

        [Serializable] struct Sample { public double wallMs, cpuMs, gpuMs, mainMs, renderMs, presentWaitMs; }
        [Serializable] sealed class Segment
        {
            public string name;
            public int frames, width, height, gpuSamples, cpuSamples, enemyCount, waypointsVisited, cameraRenderFrames;
            public double meanFps, p50Ms, p95Ms, p99Ms, gpuP95Ms, cpuP95Ms, allocatedMiB, distanceTravelled;
            public bool passed;
        }
        [Serializable] sealed class Report
        {
            public string tag, startedUtc, finishedUtc, gpu, api, cpu, colorSpace, quality, persistentDataPath, dataDirectory, routeVersion;
            public int width, height, gpuMemoryMiB, systemMemoryMiB, cascades, vSync, frameCap, firstPersonShadowOnlyRenderers, movingShadowPixels, sunShadowPixels, localLitPixels, localBlockedPixels, sunResolution, cameraWidth, cameraHeight, branch;
            public float shadowDistance, renderWidthScale, renderHeightScale;
            public double warmupSeconds, sampleSeconds;
            public bool completed, frameTimingEnabled, localLightCastsShadows, visualPassed, dynamicResolution, routeCompleted;
            public List<Segment> segments = new List<Segment>();
        }
    }
}