[Core] Fixes #526

Downloading and extracting package source archives now requires that a Module from the Package has been used in the build. This means that there may be packages with source archives that don't get downloaded on some platforms.
Note that the concurrent download seen in previous alphas of v2.0 is no longer present, since the downloads must now be deferred until a sequential part of the build step.

Author: markfinal

Bam.Core/BamState.cs

@@ -129,8 +129,6 @@ public BamState()
             this.VersionString = productVersion;
             this.TargetFrameworkVersion = targetFrameworkName;
 
-            this.preBuildTasks = new Array<System.Threading.Tasks.Task>();
-
             this.BuildStartTime = System.Diagnostics.Process.GetCurrentProcess().StartTime;
         }
 
@@ -174,58 +172,5 @@ public BamState()
         int? ISemanticVersion.MinorVersion => Version.Minor;
 
         int? ISemanticVersion.PatchVersion => Version.Build;
-
-        private readonly Array<System.Threading.Tasks.Task> preBuildTasks;
-
-        /// <summary>
-        /// Append a task from an async method that must be completed before builds start.
-        /// </summary>
-        /// <param name="task"></param>
-        public void
-        AppendPreBuildTask(
-            System.Threading.Tasks.Task task)
-        {
-            lock (this.preBuildTasks)
-            {
-                Log.DebugMessage($"Adding task {task.ToString()}");
-                this.preBuildTasks.Add(task);
-            }
-        }
-
-        private async System.Threading.Tasks.Task
-        InternalWaitOnAllPreBuildTasks()
-        {
-            var total = this.preBuildTasks.Count;
-            var all = System.Threading.Tasks.Task.WhenAll(this.preBuildTasks.ToArray());
-            for (; ; )
-            {
-                var timer = System.Threading.Tasks.Task.Delay(1000); // every second
-                var finishedTask = await System.Threading.Tasks.Task.WhenAny(all, timer);
-                if (finishedTask.IsFaulted)
-                {
-                    throw finishedTask.Exception;
-                }
-                if (all.IsCompleted)
-                {
-                    return;
-                }
-                Log.DetailProgress($"{100 * this.preBuildTasks.Count(task => task.IsCompleted) / total}%");
-            }
-        }
-
-        /// <summary>
-        /// Wait on all prebuild tasks to complete. This is a blocking function.
-        /// </summary>
-        public void
-        WaitOnAllPreBuildTasks()
-        {
-            if (!this.preBuildTasks.Any())
-            {
-                return;
-            }
-            Log.Info($"Waiting on package source downloads to finish before the build starts...");
-            var all = this.InternalWaitOnAllPreBuildTasks();
-            all.Wait(); // safety
-        }
     }
 }

Bam.Core/EntryPoint.cs

@@ -99,7 +99,6 @@ public static void
                 PackageUtilities.CompilePackageAssembly(true);
                 PackageUtilities.LoadPackageAssembly();
             }
-            Graph.Instance.ProcessState.WaitOnAllPreBuildTasks();
 
             var packageMetaDataProfile = new TimeProfile(ETimingProfiles.PackageMetaData);
             packageMetaDataProfile.StartProfile();

Bam.Core/Graph.cs

@@ -773,6 +773,20 @@ public void
                     this.InternalValidateGraph(rank.Key, m.Requirements);
                 }
             }
+            Log.DebugMessage("Used packages:");
+            foreach (var package in this.PackageDefinitions.Where(item => item.CreatedModules().Any()))
+            {
+                Log.DebugMessage($"\t{package.Name}");
+                foreach (var module in package.CreatedModules())
+                {
+                    Log.DebugMessage($"\t\t{module.ToString()}");
+                }
+            }
+            Log.DebugMessage("Unused packages:");
+            foreach (var package in this.PackageDefinitions.Where(item => !item.CreatedModules().Any()))
+            {
+                Log.DebugMessage($"\t{package.Name}");
+            }
         }
 
         /// <summary>
@@ -837,23 +851,6 @@ public void
         {
             this.PackageDefinitions = packages;
             this.Macros.AddVerbatim("masterpackagename", this.MasterPackage.Name);
-
-            foreach (var package in packages)
-            {
-                if (null == package.Sources)
-                {
-                    continue;
-                }
-                foreach (var source in package.Sources)
-                {
-                    var fetchSourceTask = source.Fetch();
-                    if (null == fetchSourceTask)
-                    {
-                        continue;
-                    }
-                    this.ProcessState.AppendPreBuildTask(fetchSourceTask);
-                }
-            }
         }
 
         /// <summary>

Bam.Core/Module.cs

@@ -122,6 +122,15 @@ protected Module()
             this.ReasonToExecute = ExecuteReasoning.Undefined();
             this.ExecutionTask = null;
             this.EvaluationTask = null;
+
+            this.PackageDefinition.AddCreatedModule(this.GetType().Name);
+            if (null != this.PackageDefinition.Sources)
+            {
+                foreach (var source in this.PackageDefinition.Sources)
+                {
+                    source.Fetch();
+                }
+            }
         }
 
         private void

Bam.Core/Package/PackageDefinition.cs

@@ -65,6 +65,33 @@ private static void
 
         private static System.Xml.XmlReaderSettings CommonReaderSettings { get; set; }
 
+        private System.Collections.Generic.HashSet<string> ModulesCreated = new System.Collections.Generic.HashSet<string>();
+
+        /// <summary>
+        /// Enumerated unique list of module names created from this package definition.
+        /// </summary>
+        public System.Collections.Generic.IEnumerable<string> CreatedModules()
+        {
+            foreach (var m in this.ModulesCreated)
+            {
+                yield return m;
+            }
+        }
+
+        /// <summary>
+        /// Add the name of another module to the unique list.
+        /// </summary>
+        /// <param name="name">String name of the module to add.</param>
+        public void
+        AddCreatedModule(
+            string name)
+        {
+            lock (this.ModulesCreated)
+            {
+                this.ModulesCreated.Add(name);
+            }
+        }
+
         static PackageDefinition()
         {
             var xmlReaderSettings = new System.Xml.XmlReaderSettings();

Bam.Core/Package/PackageSource.cs

@@ -132,17 +132,19 @@ public PackageSource(
         /// <summary>
         /// Begin the fetch of the package source if it's not already fetched.
         /// </summary>
-        public System.Threading.Tasks.Task
+        public void
         Fetch()
         {
             if (this.AlreadyFetched)
             {
-                return null;
+                return;
             }
             switch (this.Type)
             {
                 case PackageSource.EType.Http:
-                    return this.DownloadAndExtractPackageViaHTTP();
+                    this.DownloadAndExtractPackageViaHTTP().Wait();
+                    this.AlreadyFetched = true;
+                    return;
             }
             throw new Exception($"Unhandled package source type, '{this.Type.ToString()}'");
         }

Changelog.txt

@@ -1,3 +1,5 @@
+03-Apr-2019 Fixes #526. Downloading and extracting package source archives now requires that a Module from the Package has been used in the build. This means that there may be packages with source archives that don't get downloaded on some platforms.
+
 09-Mar-2019 Fixes #525. Resolving a package, when there is both an unversioned and (many) versioned candidates no longer fails with a null dereference. I believe this is a rare situation - usually all candidates for a package will be versioned when there is more than one.
 
 09-Mar-2019 Fixes #524. Upgraded NuGet dependencies: