Merge remote-tracking branch 'origin/master' into sym-2.107

payload/reggae/backend/make.d

@@ -75,9 +75,9 @@ struct Makefile {
     //includes rerunning reggae
     string output() @safe {
 
-        import std.array: join, replace;
+        import std.array: join;
         import std.range: chain;
-        import std.algorithm: sort, uniq, map;
+        import std.algorithm: sort, uniq;
 
         auto ret = simpleOutput;
 
@@ -87,11 +87,15 @@ struct Makefile {
             // add a dependency on the Makefile to reggae itself and the build description,
             // but only if not exporting a build
             auto srcDirs = _srcDirs.sort.uniq;
+            const flattenedInputs = chain(options.reggaeFileDependencies, srcDirs).join(" ");
             const rerunLine = "\t" ~ options.rerunArgs.join(" ") ~ "\n";
 
-            ret ~= fileName() ~ ": " ~ chain(options.reggaeFileDependencies, srcDirs.save).join(" ") ~ "\n";
+            ret ~= fileName() ~ ": " ~ flattenedInputs ~ "\n";
             ret ~= rerunLine;
-            ret ~= "\n" ~ srcDirs.save.map!(d => d ~ ":" ~ "\n" ~ rerunLine).join("\n");
+            // Add a dummy target for all Makefile inputs, with an empty recipe, just so
+            // that re-generating the Makefile isn't blocked by removed/renamed inputs.
+            // See ninja.d for more details.
+            ret ~= "\n" ~ flattenedInputs ~ ": ;\n";
         }
 
         return replaceEnvVars(ret);

payload/reggae/backend/ninja.d

@@ -42,29 +42,28 @@ struct Ninja {
         import std.algorithm: sort, uniq, map;
         import std.range: chain, only;
 
-        auto srcDirs = _srcDirs.sort.uniq;
-        const files = flattenEntriesInBuildLine(
-            chain(_options.reggaeFileDependencies, srcDirs).array
-        );
-        auto paramLines = _options.oldNinja ? [] : ["pool = console"];
-
         const(NinjaEntry)[] rerunEntries() {
             // if exporting the build system, don't include rerunning reggae
             if(_options.export_)
                 return [];
-            auto rerun = NinjaEntry("build build.ninja: _rerun | " ~ files, paramLines);
-            // the reason this is needed is because source directories
-            // can be deleted or renamed. If they are, ninja will
-            // complain about the missing directories since they are a
-            // dependency of the build.ninja file for rerunning
-            // reggae.  So we consider them phony targets with no
-            // dependencies to make that not happen and ninja will
-            // rerun reggae and change the list of source directories
-            // accordingly.
-            auto phonySrcDirs = srcDirs
-                .map!(a => NinjaEntry("build " ~ a.escapeBuildLine ~ ": phony"))
-                ;
-            return chain(rerun.only, phonySrcDirs).array;
+
+            auto srcDirs = _srcDirs.sort.uniq;
+            const flattenedInputs = flattenEntriesInBuildLine(
+                chain(_options.reggaeFileDependencies, srcDirs).array
+            );
+            auto paramLines = _options.oldNinja ? [] : ["pool = console"];
+
+            auto rerun = NinjaEntry("build build.ninja: _rerun | " ~ flattenedInputs, paramLines);
+            // the reason this is needed is because inputs (files and
+            // source directories) can be deleted or renamed. If they are,
+            // ninja will complain about the missing files/directories
+            // since they are a dependency of the build.ninja file for
+            // rerunning reggae.  So use a dummy phony target 'outputting'
+            // all build.ninja inputs, so that ninja will happily rerun
+            // reggae, which will generate a new build.ninja with an
+            // updated list of build.ninja inputs.
+            auto phonyInputs = NinjaEntry("build " ~ flattenedInputs ~ ": phony");
+            return [rerun, phonyInputs];
         }
 
         const defaultOutputs = _build.defaultTargetsOutputs(_projectPath);
@@ -377,7 +376,7 @@ private:
         import std.algorithm: map;
         import std.array: join, replace;
         return entries
-            .map!escapeBuildLine
+            .map!escapePathInBuildLine
             .join(" ");
     }
 
@@ -405,9 +404,9 @@ private:
     }
 }
 
-private string escapeBuildLine(in string line) @safe pure {
+private string escapePathInBuildLine(in string path) @safe pure {
     import std.array: replace;
-    return line.replace(":", "$:").replace(" ", "$ ");
+    return path.replace(":", "$:").replace(" ", "$ ");
 }
 
 struct NinjaEntry {

payload/reggae/dub/interop/dublib.d

@@ -29,16 +29,30 @@ static this() nothrow {
     }
 }
 
+private struct DubConfigurations {
+    string[] configurations;
+    string default_;
+    string test; // special `dub test` config
+
+    bool haveTestConfig() @safe @nogc pure nothrow scope const {
+        return test != "";
+    }
+
+    bool isTestConfig(in string config) @safe @nogc pure nothrow scope const {
+        return haveTestConfig && config == test;
+    }
+}
 
 package struct Dub {
-    import reggae.dub.interop.configurations: DubConfigurations;
     import reggae.dub.info: DubInfo;
     import reggae.options: Options;
     import dub.dub: DubClass = Dub;
+    import dub.generators.generator: GeneratorSettings;
 
     private DubClass _dub;
     private const string[] _extraDFlags;
     private const(Options) _options;
+    private GeneratorSettings _generatorSettings;
 
     this(in Options options) @trusted {
         import reggae.path: buildPath;
@@ -48,6 +62,7 @@ package struct Dub {
         _options = options;
         _dub = fullDub(options.projectPath);
         _extraDFlags = options.dflags.dup;
+        _generatorSettings = getGeneratorSettings(options);
     }
 
     const(Options) options() @safe @nogc pure nothrow const return scope {
@@ -76,14 +91,12 @@ package struct Dub {
         DubInfo[string] ret;
 
         const configs = dubConfigurations(output);
-        const haveTestConfig = configs.test != "";
         bool atLeastOneConfigOk;
         Exception dubInfoFailure;
 
         foreach(config; configs.configurations) {
-            const isTestConfig = haveTestConfig && config == configs.test;
             try {
-                ret[config] = configToDubInfo(output, config, isTestConfig);
+                ret[config] = configToDubInfo(output, config, configs.isTestConfig(config));
                 atLeastOneConfigOk = true;
             } catch(Exception ex) {
                 output.log("ERROR: Could not get info for configuration ", config, ": ", ex.msg);
@@ -102,7 +115,7 @@ package struct Dub {
         // (additionally) expose the special `dub test` config as
         // `unittest` config in the DSL (`configToDubInfo`) (for
         // `dubTest!()`, `dubBuild!(Configuration("unittest"))` etc.)
-        if(haveTestConfig && configs.test != "unittest" && configs.test in ret)
+        if(configs.haveTestConfig && configs.test != "unittest" && configs.test in ret)
             ret["unittest"] = ret[configs.test];
 
         return ret;
@@ -127,10 +140,7 @@ package struct Dub {
         return ret;
     }
 
-    imported!"reggae.dub.interop.configurations".DubConfigurations
-    dubConfigurations(O)(ref O output)
-    {
-        import reggae.dub.interop.configurations: DubConfigurations;
+    DubConfigurations dubConfigurations(O)(ref O output) {
         import reggae.io: log;
 
         output.log("Getting dub configurations");
@@ -150,12 +160,11 @@ package struct Dub {
         import std.conv: text;
 
         auto singleConfig = _options.dubConfig;
-        auto settings = getGeneratorSettings(_options);
         const allConfigs = singleConfig == "";
         // add the special `dub test` configuration (which doesn't require an existing `unittest` config)
         const lookingForUnitTestsConfig = allConfigs || singleConfig == "unittest";
         const testConfig = lookingForUnitTestsConfig
-            ? _dub.project.addTestRunnerConfiguration(settings)
+            ? _dub.project.addTestRunnerConfiguration(_generatorSettings)
             : null; // skip when requesting a single non-unittest config
 
         // error out if the test config is explicitly requested but not available
@@ -164,7 +173,7 @@ package struct Dub {
         }
 
         const haveSpecialTestConfig = testConfig.length && testConfig != "unittest";
-        const defaultConfig = _dub.project.getDefaultConfiguration(settings.platform);
+        const defaultConfig = _dub.project.getDefaultConfiguration(_generatorSettings.platform);
 
         // A violation of the Law of Demeter caused by a dub bug.
         // Otherwise _dub.project.configurations would do, but it fails for one
@@ -174,7 +183,7 @@ package struct Dub {
             .rootPackage
             .recipe
             .configurations
-            .filter!(c => c.matchesPlatform(settings.platform))
+            .filter!(c => c.matchesPlatform(_generatorSettings.platform))
             .map!(c => c.name)
             ;
 
@@ -252,7 +261,7 @@ package struct Dub {
 
     private DubInfo configToDubInfo(in string config = "") @trusted /*dub*/ {
         auto generator = new InfoGenerator(_dub.project, _extraDFlags);
-        auto settings = getGeneratorSettings(_options);
+        auto settings = _generatorSettings;
         settings.config = config;
         generator.generate(settings);
         return DubInfo(generator.dubPackages, _options.dup);
@@ -267,9 +276,33 @@ package struct Dub {
 // module that don't do that on purpose for speed reasons.
 auto fullDub(in string projectPath) @trusted {
     import dub.dub: DubClass = Dub;
+    import dub.packagemanager: PackageManager;
     import dub.internal.vibecompat.inet.path: NativePath;
 
-    auto dub = new DubClass(projectPath);
+    // Cache the PackageManager.
+    // A reggaefile.d with lots of dub{Package,Dependant} targets benefits from
+    // this, also depending on the size of the dub packages cache.
+    static class DubWithCachedPackageManager : DubClass {
+        this(string rootPath) {
+            super(rootPath);
+        }
+
+        override PackageManager makePackageManager() const {
+            static PackageManager cachedPM = null;
+            if (!cachedPM) {
+                // The PackageManager wants a path to a local directory, for an
+                // implicit `<local>/.dub/packages` repo. The base
+                // implementation uses the dub root project directory; use the
+                // reggae project directory as our local root.
+                import reggae.config: options;
+                auto localRoot = NativePath(options.projectPath);
+                cachedPM = new PackageManager(localRoot, m_dirs.userPackages, m_dirs.systemSettings, false);
+            }
+            return cachedPM;
+        }
+    }
+
+    auto dub = new DubWithCachedPackageManager(projectPath);
     dub.packageManager.getOrLoadPackage(NativePath(projectPath));
     dub.loadPackage();
     dub.project.validate();

payload/reggae/io.d

@@ -20,7 +20,8 @@ void log(O, T...)(auto ref O output, auto ref T args) {
 private string secondsSinceStartString() @safe {
     import std.string: rightJustify;
     import std.conv: to;
-    return ("+" ~ (sinceStart / 1000.0).to!string).rightJustify(8, ' ');
+    import std.format: format;
+    return ("+" ~ (sinceStart / 1000.0).format!"%03.3f"()).rightJustify(8, ' ');
 }
 
 

tests/it/runtime/dependencies.d

@@ -224,10 +224,14 @@ unittest {
                 ]
             );
 
-            // but do write configuration if dub cfg has changed
-            auto lines = runIt("-d myvar=foo");
-            srcLine.should.not.be in lines;
-            cfgLine.should.be in lines;
+            // we cache dub's PackageManager per-thread; use a new thread to make sure the changed .sdl is reloaded
+            import core.thread: Thread;
+            new Thread(() {
+                // but do write configuration if dub cfg has changed
+                auto lines = runIt("-d myvar=foo");
+                srcLine.should.not.be in lines;
+                cfgLine.should.be in lines;
+            }).start().join();
         }
     }
 }

tests/it/runtime/issues.d

@@ -520,5 +520,44 @@ version(DigitalMars) {
                 mixin(backend).shouldFailToExecute.shouldNotContain(msg);
             }
         }
+
+        @("rerun.deleted.file." ~ backend)
+        @Tags(backend)
+        unittest {
+            import std.file: remove;
+
+            with(immutable ReggaeSandbox()) {
+
+                writeFile(
+                    "imported.d",
+                    q{
+                        import reggae;
+                        enum myTarget = Target.phony("dummy", "");
+                    }
+                );
+                writeFile(
+                    "reggaefile.d",
+                    q{
+                        import imported, reggae;
+                        mixin build!myTarget;
+                    }
+                );
+
+                runReggae("-b", backend, "--verbose");
+                mixin(backend).shouldExecuteOk;
+
+                // delete imported.d and make sure reggae gets rerun
+                remove(inSandboxPath("imported.d"));
+
+                static if(backend == "ninja")
+                    enum msg = "missing and no known rule";
+                else static if(backend == "make")
+                    enum msg = "No rule to make target";
+                else
+                    static assert(false, "unknown backend");
+
+                mixin(backend).shouldFailToExecute.shouldNotContain(msg);
+            }
+        }
     }
 }