1 /** Software package builder.
2  *
3  * Run as: dmd -debug -g -I.. -i xy.d -version=main && ./xy dmd phobos dub DCD DustMite
4  *
5  * TODO: Support `xy REPO_URL`. Try for instance git@github.com:rui314/mold.git
6  * or https://github.com/rui314/mold.git. Supports matching repo with registered
7  * repos where repo name is stripped ending slash before matching.
8  *
9  * TODO: resolution-pyramid sample chronologically joined commits over dmd, phobos
10  * TODO: Experiment with indentation of stdout and stderr lines
11  * TODO: Show memory usage using https://code.dlang.org/packages/resusage
12  * TODO: Add install using installPrefix
13  * TODO: Skip building if fingerprint hasn’t changed unless --force -f is used. Put HEAD git-fingerprint directly under installPrefix
14  * TODO: Delete installPrefix prior to installation
15  * TODO: Add argparse --build=debug|release
16  * TODO: Pick compiler dmd if --build=debug
17  * TODO: Pick compiler ldmd2 if --build=release
18  * TODO: --test
19  */
20 module nxt.xy;
21 
22 import std.path : absolutePath, expandTilde, buildPath;
23 import std.stdio : writeln;
24 import std.file : chdir, exists, mkdir, FileException;
25 import std.process : Config, spawnProcess, wait, Pid;
26 import std.algorithm : canFind, joiner;
27 import std.conv : to;
28 import std.exception : enforce;
29 import std.getopt : getopt;
30 
31 import nxt.fs;
32 import nxt.cmd;
33 
34 @safe:
35 
36 /// Spec Name.
37 struct Name { string value; alias value this; }
38 
39 /// D `version` symbol.
40 alias DlangVersionName = string;
41 
42 struct RemoteSpec
43 {
44     URL url;
45     string remoteName;
46     string[] repoBranches;
47 }
48 
49 version(main)
50     int main(string[] args)
51     {
52         return buildN(args);
53     }
54 
55 /** Build the packages `packages`.
56  */
57 int buildN(scope const string[] packages)
58 {
59     const bool echo = true; // TODO: getopt flag being negation of make-like flag -s, --silent, --quiet
60     const BuildOption[] bos;
61     auto builds = makeSpecs(echo);
62     foreach (const name; packages)
63         if (Spec* buildPtr = Name(name) in builds)
64             (*buildPtr).go(bos, echo);
65 	return 0;                   // TODO: propagate exit code
66 }
67 
68 const DlangVersionName[] versionNames = ["cli"];
69 const string[] dflagsLDCRelease = ["-m64", "-O3", "-release", "-boundscheck=off", "-enable-inlining", "-flto=full"];
70 
71 enum BuildOption
72 {
73 	debugMode,            /// Compile in debug mode (enables contracts, -debug)
74 	releaseMode,          /// Compile in release mode (disables assertions and bounds checks, -release)
75 	ltoMode,              /// Compile using LTO
76 	pgoMode,              /// Compile using PGO
77 	coverage,             /// Enable code coverage analysis (-cov)
78 	debugInfo,            /// Enable symbolic debug information (-g)
79 	debugInfoC,           /// Enable symbolic debug information in C compatible form (-gc)
80 	alwaysStackFrame,     /// Always generate a stack frame (-gs)
81 	stackStomping,        /// Perform stack stomping (-gx)
82 	inline,               /// Perform function inlining (-inline)
83 	optimize,             /// Enable optimizations (-O)
84 	profile,              /// Emit profiling code (-profile)
85 	unittests,            /// Compile unit tests (-unittest)
86 	verbose,              /// Verbose compiler output (-v)
87 	ignoreUnknownPragmas, /// Ignores unknown pragmas during compilation (-ignore)
88 	syntaxOnly,           /// Don't generate object files (-o-)
89 	warnings,             /// Enable warnings (-wi)
90 	warningsAsErrors,     /// Treat warnings as errors (-w)
91 	ignoreDeprecations,   /// Do not warn about using deprecated features (-d)
92 	deprecationWarnings,  /// Warn about using deprecated features (-dw)
93 	deprecationErrors,    /// Stop compilation upon usage of deprecated features (-de)
94 	property,             /// DEPRECATED: Enforce property syntax (-property)
95 	profileGC,            /// Profile runtime allocations
96 	pic,                  /// Generate position independent code
97 	betterC,              /// Compile in betterC mode (-betterC)
98 	lowmem,               /// Compile in lowmem mode (-lowmem)
99 }
100 
101 Spec[Name] makeSpecs(bool echo)
102 {
103     typeof(return) specs;
104     {
105         const name = "vox";
106         Spec spec =
107         { homePage : URL("https://github.com/MrSmith33/"~name~"/"),
108           name : Name(name),
109           url : URL("https://github.com/MrSmith33/"~name~".git"),
110           compiler : ExePath("ldc2"),
111           versionNames : ["cli"],
112           sourceFilePaths : [Path("main.d")],
113           outFilePath : Path(name~".out"),
114         };
115         specs[spec.name] = spec;
116     }
117     {
118         const name = "dmd";
119         Spec spec =
120         { homePage : URL("http://dlang.org/"),
121           name : Name(name),
122           url : URL("https://github.com/dlang/"~name~".git"),
123           // nordlow/selective-unittest-2
124           // nordlow/check-self-assign
125           // nordlow/check-unused
126           extraRemoteSpecs : [RemoteSpec(URL("https://github.com/nordlow/dmd.git"), "nordlow",
127                                          ["relax-alias-assign",
128 										 "traits-isarray-take2",
129                                           "diagnose-padding",
130                                           "aliasseq-bench",
131                                           "add-traits-hasAliasing",
132                                           "unique-qualifier"]),
133                               // RemoteSpec(URL("https://github.com/UplinkCoder/dmd.git"), "skoch", ["newCTFE_upstream"]),
134           ],
135           // patches : [Patch(Path("new-ctfe.patch"), 1, echo)],
136           buildFlagsDefault : ["PIC=1"],
137           buildFlagsByBO : [BuildOption.debugMode : ["ENABLE_DEBUG=1"],
138                             BuildOption.releaseMode : ["ENABLE_RELASE=1"]],
139         };
140         specs[spec.name] = spec;
141     }
142     {
143         const name = "phobos";
144         Spec spec =
145         { homePage : URL("http://dlang.org/"),
146           name : Name(name),
147           url : URL("https://github.com/dlang/"~name~".git"),
148           extraRemoteSpecs : [RemoteSpec(URL("https://github.com/nordlow/phobos.git"), "nordlow", ["faster-formatValue"])],
149         };
150         specs[spec.name] = spec;
151     }
152     {
153         const name = "dub";
154         Spec spec =
155         { homePage : URL("http://dlang.org/"),
156           name : Name(name),
157           url : URL("https://github.com/dlang/"~name~".git"),
158         };
159         specs[spec.name] = spec;
160     }
161     {
162         const name = "DustMite";
163         Spec spec =
164         { homePage : URL("https://github.com/CyberShadow/"~name~"/wiki"),
165           name : Name(name),
166           compiler : ExePath("ldmd2"),
167           sources : ["dustmite.d", "polyhash.d", "splitter.d"],
168           url : URL("https://github.com/CyberShadow/"~name~".git"),
169         };
170         specs[spec.name] = spec;
171     }
172     {
173         const name = "DCD";
174         Spec spec =
175         { name : Name(name),
176           url : URL("https://github.com/dlang-community/"~name~".git"),
177           buildFlagsDefault : ["ldc"],
178           buildFlagsByBO : [BuildOption.debugMode : ["debug"],
179                             BuildOption.releaseMode : ["ldc"]],
180         };
181         specs[spec.name] = spec;
182     }
183     {
184         const name = "mold";
185         Spec spec =
186         { name : Name(name),
187           url : URL("https://github.com/rui314/"~name~".git"),
188         };
189         specs[spec.name] = spec;
190     }
191     return specs;
192 }
193 
194 /** Launch/Execution specification.
195  */
196 @safe struct Spec
197 {
198     URL homePage;
199     Name name;
200 
201     // Git sources
202     URL url;
203     string remote = "origin";
204     string branch;
205     RemoteSpec[] extraRemoteSpecs;
206     Patch[] patches;
207 
208     ExePath compiler;
209     string[] sources;
210     string[] buildFlagsDefault;
211     string[][BuildOption] buildFlagsByBO;
212     DlangVersionName[] versionNames;
213     Path[] sourceFilePaths;
214     Path outFilePath;
215     uint jobCount;
216     bool recurseSubModulesFlag = true;
217 
218     void go(scope ref const BuildOption[] bos, in bool echo) scope
219     {
220         const dlDirName = DirName(".cache/repos");
221         const dlDir = DirPath(("~/" ~ dlDirName ~ "/").expandTilde.buildPath(name));
222         writeln();
223         fetch(bos, dlDir, echo);
224         build(bos, dlDir, echo);
225     }
226 
227     void fetch(scope ref const BuildOption[] bos, in DirPath dlDir, in bool echo) scope
228     {
229         import core.thread : Thread;
230         import core.time : dur;
231         auto repo = Repository(url, dlDir, echo);
232         // TODO: use waitAllInSequence();
233         enforce(!repo.cloneOrRefresh(recurseSubModulesFlag, remote, branch).wait());
234         enforce(!repo.clean().wait());
235         string[] remoteNames;
236         foreach (const spec; extraRemoteSpecs)
237         {
238             remoteNames ~= spec.remoteName;
239             const statusIgnored = repo.remoteRemove(spec.remoteName).wait(); // ok if already removed
240             enforce(!repo.remoteAdd(spec.url, spec.remoteName).wait());
241         }
242         enforce(!repo.fetch(remoteNames).wait());
243         foreach (const spec; extraRemoteSpecs)
244             foreach (const branch; spec.repoBranches)
245                 enforce(!repo.merge([spec.remoteName~"/"~branch], ["--no-edit", "-Xignore-all-space"]).wait());
246         foreach (ref patch; patches)
247             enforce(!patch.applyIn(dlDir).wait());
248     }
249 
250     string[] bosFlags(scope ref const BuildOption[] bos) const scope pure nothrow
251     {
252         typeof(return) result;
253         foreach (bo; bos)
254             result ~= buildFlagsByBO[bo];
255         return result;
256     }
257 
258     string[] allBuildFlags(scope ref const BuildOption[] bos) const scope pure nothrow
259     {
260         return buildFlagsDefault ~ bosFlags(bos) ~ jobFlags() ~ sources;
261     }
262 
263     void build(scope ref const BuildOption[] bos, in DirPath workDir, in bool echo) const scope
264     {
265         string[] args;
266         Config config = Config.none;
267         size_t maxOutput = size_t.max;
268 
269         chdir(workDir);         // TODO: replace with absolute paths in exists or dir.exists
270 
271         // TODO: when serveral of this exists ask user
272         if (exists("Makefile"))
273             args = ["make", "-f", "Makefile"] ~ allBuildFlags(bos);
274         else if (exists("makefile"))
275             args = ["make", "-f", "makefile"] ~ allBuildFlags(bos);
276         else if (exists("posix.mak"))
277             args = ["make", "-f", "posix.mak"] ~ allBuildFlags(bos);
278         else if (exists("dub.sdl") ||
279                  exists("dub.json"))
280             args = ["dub", "build"];
281         else
282         {
283             args = [compiler];
284             if (bos.canFind(BuildOption.releaseMode) &&
285                 (compiler[].canFind("ldc2") ||
286                  compiler[].canFind("ldmd2")))
287                 args ~= dflagsLDCRelease; // TODO: make declarative
288             foreach (const ver; versionNames)
289                 args ~= ("-d-version=" ~ ver);
290             args ~= sources;
291         }
292         /* ${DC} -release -boundscheck=off dustmite.d splitter.d polyhash.d */
293         /*       mkdir -p ${DMD_EXEC_PREFIX} */
294         /* mv dustmite ${DMD_EXEC_PREFIX} */
295 
296         enforce(args.length, "Could not deduce build command");
297 
298         // TOOD: wrap in Spawn
299         writeln("Building as `", args.joiner(" "), "` ...");
300         string[string] env_;
301         /* TODO: add a wrapper spawnProcess1 called by Patch.applyIn,
302          * Repository.spawn and here that sets stdout, stderr based on bool
303          * echo. */
304         scope Pid pid = spawnProcess(args, env_, config, workDir);
305         const status = pid.wait();
306         if (status != 0)
307             writeln("Compilation failed:\n");
308         if (status != 0)
309             writeln("Compilation successful:\n");
310     }
311 
312     string[] jobFlags() const scope pure nothrow
313     {
314         return jobCount ? ["-j"~jobCount.to!string] : [];
315     }
316 }
317 
318 /// Relaxed variant of `mkdir`.
319 void mkdirRelaxed(in const(char)[] pathname)
320 {
321     try
322         mkdir(pathname);
323     catch (FileException e) {}  // TODO: avoid need for throwing
324 }
325 
326 int[] waitAllInSequence(scope Pid[] pids...)
327 {
328     typeof(return) statuses;
329     foreach (pid; pids)
330         statuses ~= pid.wait();
331     return statuses;
332 }
333 
334 int[] waitAllInParallel(scope Pid[] pids...)
335 {
336     assert(0, "TODO: implement");
337 }