1 /** Universal Build System. 2 * 3 * TODO: Support `$0 REPO_URL`. Try for instance git@github.com:rui314/mold.git 4 * or https://github.com/rui314/mold.git. Supports matching repo with registered 5 * repos where repo name is stripped ending slash before matching. 6 * 7 * TODO: Delete installPrefix prior to installation 8 * 9 * TODO: Support `xy REPO_URL`. Try for instance git@github.com:rui314/mold.git 10 * or https://github.com/rui314/mold.git. Supports matching repo with registered 11 * repos where repo name is stripped ending slash before matching. 12 * 13 */ 14 module xdag; 15 16 // version = main; 17 18 debug import std.stdio : writeln; 19 import std.process : Config, spawnProcess, wait, Pid; 20 21 import nxt.path; 22 import nxt.file : homeDir; 23 import nxt.git : RepoURL; 24 25 private alias ProcessEnvironment = string[string]; 26 27 version (main) { 28 int main(string[] args) { 29 return buildN(args[1 .. $]); 30 } 31 } 32 33 /// Spec Name. 34 struct Name { string value; alias value this; } 35 36 /// D `version` symbol. 37 alias DlangVersionName = string; 38 39 struct RemoteSpec { 40 RepoURL url; 41 string remoteName; 42 string[] repoBranches; 43 } 44 45 void ewriteln(S...)(S args) @trusted { 46 import std.stdio : writeln, stderr; 47 return stderr.writeln(args); 48 } 49 50 /** Build the packages `packages`. 51 */ 52 int buildN(scope const string[] packages) { 53 const bool echo = true; // std.getopt: TODO: getopt flag being negation of make-like flag -s, --silent, --quiet 54 const BuildOption[] bos; 55 auto builds = makeSpecs(echo); 56 foreach (const name; packages) 57 if (Spec* buildPtr = Name(name) in builds) { 58 (*buildPtr).go(bos, echo); 59 } 60 else { 61 ewriteln("No spec named ", Name(name), " found"); 62 } 63 return 0; /+ TODO: propagate exit code +/ 64 } 65 66 const DlangVersionName[] versionNames = ["cli"]; 67 const string[] dflagsLDCRelease = ["-m64", "-O3", "-release", "-boundscheck=off", "-enable-inlining", "-flto=full"]; 68 69 /++ Build option. 70 +/ 71 enum BuildOption { 72 debugMode, /// Compile in debug mode (enables contracts, -debug) 73 releaseMode, /// Compile in release mode (disables assertions and bounds checks, -release) 74 ltoMode, /// Compile using LTO 75 pgoMode, /// Compile using PGO 76 coverage, /// Enable code coverage analysis (-cov) 77 debugInfo, /// Enable symbolic debug information (-g) 78 debugInfoC, /// Enable symbolic debug information in C compatible form (-gc) 79 alwaysStackFrame, /// Always generate a stack frame (-gs) 80 stackStomping, /// Perform stack stomping (-gx) 81 inline, /// Perform function inlining (-inline) 82 optimize, /// Enable optimizations (-O) 83 profile, /// Emit profiling code (-profile) 84 unittests, /// Compile unit tests (-unittest) 85 verbose, /// Verbose compiler output (-v) 86 ignoreUnknownPragmas, /// Ignores unknown pragmas during compilation (-ignore) 87 syntaxOnly, /// Don't generate object files (-o-) 88 warnings, /// Enable warnings (-wi) 89 warningsAsErrors, /// Treat warnings as errors (-w) 90 ignoreDeprecations, /// Do not warn about using deprecated features (-d) 91 deprecationWarnings, /// Warn about using deprecated features (-dw) 92 deprecationErrors, /// Stop compilation upon usage of deprecated features (-de) 93 property, /// DEPRECATED: Enforce property syntax (-property) 94 profileGC, /// Profile runtime allocations 95 pic, /// Generate position independent code 96 betterC, /// Compile in betterC mode (-betterC) 97 lowmem, /// Compile in lowmem mode (-lowmem) 98 } 99 100 alias BuildFlagsByOption = string[][BuildOption]; 101 102 Spec[Name] makeSpecs(bool echo) { 103 typeof(return) specs; 104 { 105 const name = "vox"; 106 Spec spec = 107 { homePage : URL("https://github.com/MrSmith33/"~name~"/"), 108 name : Name(name), 109 url : RepoURL("https://github.com/MrSmith33/"~name~".git"), 110 compiler : FilePath("ldc2"), 111 versionNames : ["cli"], 112 sourceFilePaths : [Path("main.d")], 113 outFilePath : Path(name~".out"), 114 }; 115 specs[spec.name] = spec; 116 } 117 { 118 const name = "dmd"; 119 Spec spec = 120 { homePage : URL("http://dlang.org/"), 121 name : Name(name), 122 url : RepoURL("https://github.com/dlang/"~name~".git"), 123 // nordlow/selective-unittest-2 124 // nordlow/check-self-assign 125 // nordlow/check-unused 126 extraRemoteSpecs : [RemoteSpec(RepoURL("https://github.com/nordlow/dmd.git"), "nordlow", 127 ["relax-alias-assign", 128 "traits-isarray-take2", 129 "diagnose-padding", 130 "aliasseq-bench", 131 "add-traits-hasAliasing", 132 "unique-qualifier"]), 133 // RemoteSpec(RepoURL("https://github.com/UplinkCoder/dmd.git"), "skoch", ["newCTFE_upstream"]), 134 ], 135 // patches : [Patch(Path("new-ctfe.patch"), 1, echo)], 136 buildFlagsDefault : ["PIC=1"], 137 buildFlagsByOption : [BuildOption.debugMode : ["ENABLE_DEBUG=1"], 138 BuildOption.releaseMode : ["ENABLE_RELASE=1"]], 139 }; 140 specs[spec.name] = spec; 141 } 142 { 143 const name = "phobos"; 144 Spec spec = 145 { homePage : RepoURL("http://dlang.org/"), 146 name : Name(name), 147 url : RepoURL("https://github.com/dlang/"~name~".git"), 148 extraRemoteSpecs : [RemoteSpec(RepoURL("https://github.com/nordlow/phobos.git"), "nordlow", ["faster-formatValue"])], 149 }; 150 specs[spec.name] = spec; 151 } 152 { 153 const name = "dub"; 154 Spec spec = 155 { homePage : RepoURL("http://dlang.org/"), 156 name : Name(name), 157 url : RepoURL("https://github.com/dlang/"~name~".git"), 158 }; 159 specs[spec.name] = spec; 160 } 161 { 162 const name = "DustMite"; 163 Spec spec = 164 { homePage : RepoURL("https://github.com/CyberShadow/"~name~"/wiki"), 165 name : Name(name), 166 compiler : FilePath("ldmd2"), 167 sources : ["dustmite.d", "polyhash.d", "splitter.d"], 168 url : RepoURL("https://github.com/CyberShadow/"~name~".git"), 169 }; 170 specs[spec.name] = spec; 171 } 172 { 173 const name = "DCD"; 174 Spec spec = 175 { name : Name(name), 176 url : RepoURL("https://github.com/dlang-community/"~name~".git"), 177 buildFlagsDefault : ["ldc"], 178 buildFlagsByOption : [BuildOption.debugMode : ["debug"], 179 BuildOption.releaseMode : ["ldc"]], 180 }; 181 specs[spec.name] = spec; 182 } 183 { 184 const name = "mold"; 185 Spec spec = 186 { name : Name(name), 187 url : RepoURL("https://github.com/rui314/"~name~".git"), 188 }; 189 specs[spec.name] = spec; 190 } 191 return specs; 192 } 193 194 /** Launch/Execution specification. 195 */ 196 @safe struct Spec { 197 import std.exception : enforce; 198 import std.algorithm : canFind, joiner; 199 import std.file : chdir, exists, mkdir; 200 import nxt.patching : Patch; 201 202 URL homePage; 203 Name name; 204 205 // Git sources 206 RepoURL url; 207 string gitRemote = "origin"; 208 string gitBranch; 209 RemoteSpec[] extraRemoteSpecs; 210 Patch[] patches; 211 212 FilePath compiler; 213 string[] sources; 214 string[] buildFlagsDefault; 215 BuildFlagsByOption buildFlagsByOption; 216 DlangVersionName[] versionNames; 217 Path[] sourceFilePaths; 218 Path outFilePath; 219 uint jobCount; 220 bool recurseSubModulesFlag = true; 221 222 void go(scope ref const BuildOption[] bos, in bool echo) scope @trusted { // TODO: -dip1000 without @trusted 223 const dlRoot = homeDir.buildPath(DirPath(".cache/repos")); 224 const dlDir = dlRoot.buildPath(DirName(name)); 225 writeln(); 226 /+ TODO: create build DAG and link these steps: +/ 227 fetch(bos, dlDir, echo); 228 prebuild(bos, dlDir, echo); 229 build(bos, dlDir, echo); 230 } 231 232 void fetch(scope ref const BuildOption[] bos, in DirPath dlDir, in bool echo) scope @trusted { // TODO: -dip1000 without @trusted 233 import nxt.git : RepositoryAndDir; 234 import core.thread : Thread; 235 auto rad = RepositoryAndDir(url, dlDir, echo); 236 /+ TODO: use waitAllInSequence(); +/ 237 enforce(!rad.cloneOrResetHard(recurseSubModulesFlag, gitRemote, gitBranch).wait()); 238 enforce(!rad.clean().wait()); 239 string[] remoteNames; 240 foreach (const spec; extraRemoteSpecs) 241 { 242 remoteNames ~= spec.remoteName; 243 const statusIgnored = rad.remoteRemove(spec.remoteName).wait(); // ok if already removed 244 enforce(!rad.remoteAdd(spec.url, spec.remoteName).wait()); 245 } 246 enforce(!rad.fetch(remoteNames).wait()); 247 foreach (const spec; extraRemoteSpecs) 248 foreach (const gitBranch; spec.repoBranches) 249 enforce(!rad.merge([spec.remoteName~"/"~gitBranch], ["--no-edit", "-Xignore-all-space"]).wait()); 250 foreach (ref patch; patches) 251 enforce(!patch.applyIn(dlDir).wait()); 252 } 253 254 string[] bosFlags(scope ref const BuildOption[] bos) const scope pure nothrow { 255 typeof(return) result; 256 foreach (bo; bos) 257 result ~= buildFlagsByOption[bo]; 258 return result; 259 } 260 261 string[] allBuildFlags(scope ref const BuildOption[] bos) const scope pure nothrow { 262 return buildFlagsDefault ~ bosFlags(bos) ~ jobFlags() ~ sources; 263 } 264 265 void prebuild(scope ref const BuildOption[] bos, in DirPath workDir, in bool echo) const scope { 266 if (exists("CMakeLists.txt")) { 267 return prebuildCmake(bos, workDir, echo); 268 } 269 } 270 271 void prebuildCmake(scope ref const BuildOption[] bos, in DirPath workDir, in bool echo) const scope @trusted { // TODO: -dip1000 without @trusted 272 Config config = Config.none; 273 string[] args; 274 ProcessEnvironment env_; 275 276 chdir(workDir.str); /+ TODO: replace with absolute paths in exists or dir.exists +/ 277 278 args = ["cmake"]; 279 const ninjaAvailable = true; /+ TODO: check if ninja is available +/ 280 if (ninjaAvailable) { 281 args ~= ["-G", "Ninja"]; 282 } 283 args ~= ["."]; /+ TODO: support out-of-source (OOS) build +/ 284 writeln("Pre-Building as `", args.joiner(" "), "` ..."); 285 scope Pid pid = spawnProcess(args, env_, config, workDir.str); 286 const status = pid.wait(); 287 if (status != 0) 288 writeln("Compilation failed:\n"); 289 if (status != 0) 290 writeln("Compilation successful:\n"); 291 } 292 293 void build(scope ref const BuildOption[] bos, in DirPath workDir, in bool echo) const scope @trusted { // TODO: -dip1000 without @trusted 294 Config config = Config.none; 295 296 chdir(workDir.str); /+ TODO: replace with absolute paths in exists or dir.exists +/ 297 298 /+ TODO: create build DAG and link these steps: +/ 299 300 string[] args; 301 ProcessEnvironment env_; 302 303 /+ TODO: when serveral of this exists ask user +/ 304 if (exists("Makefile")) 305 args = ["make", "-f", "Makefile"] ~ allBuildFlags(bos); 306 else if (exists("makefile")) 307 args = ["make", "-f", "makefile"] ~ allBuildFlags(bos); 308 else if (exists("posix.mak")) 309 args = ["make", "-f", "posix.mak"] ~ allBuildFlags(bos); 310 else if (exists("dub.sdl") || 311 exists("dub.json")) 312 args = ["dub", "build"]; 313 else 314 { 315 if (!compiler) { 316 if (bos.canFind(BuildOption.debugMode)) 317 args = [FilePath("dmd").str]; 318 else if (bos.canFind(BuildOption.releaseMode)) 319 args = [FilePath("ldc2").str]; 320 else 321 args = [FilePath("dmd").str]; // default to faster dmd 322 } else { 323 args = [compiler.str]; 324 } 325 if (bos.canFind(BuildOption.releaseMode) && 326 (compiler.str.canFind("ldc2") || 327 compiler.str.canFind("ldmd2"))) 328 args ~= dflagsLDCRelease; /+ TODO: make declarative +/ 329 foreach (const ver; versionNames) 330 args ~= ("-d-version=" ~ ver); 331 args ~= sources; 332 } 333 /* ${DC} -release -boundscheck=off dustmite.d splitter.d polyhash.d */ 334 /* mkdir -p ${DMD_EXEC_PREFIX} */ 335 /* mv dustmite ${DMD_EXEC_PREFIX} */ 336 337 enforce(args.length, "Could not deduce build command"); 338 339 /+ TODO: Use: import nxt.cmd : spawn; +/ 340 writeln("Building as `", args.joiner(" "), "` ..."); 341 /* TODO: add a wrapper spawnProcess1 called by Patch.applyIn, 342 * Repository.spawn and here that sets stdout, stderr based on bool 343 * echo. */ 344 scope Pid pid = spawnProcess(args, env_, config, workDir.str); 345 const status = pid.wait(); 346 if (status != 0) 347 writeln("Compilation failed:\n"); 348 if (status != 0) 349 writeln("Compilation successful:\n"); 350 } 351 352 string[] jobFlags() const scope pure nothrow { 353 import std.conv : to; 354 return jobCount ? ["-j"~jobCount.to!string] : []; 355 } 356 } 357 358 int[] waitAllInSequence(scope Pid[] pids...) { 359 typeof(return) statuses; 360 foreach (pid; pids) 361 statuses ~= pid.wait(); 362 return statuses; 363 } 364 365 int[] waitAllInParallel(scope Pid[] pids...) { 366 assert(0, "TODO: implement and use in place of waitAllInSequence"); 367 }