From fc886dcca3453b316b401870abfe25ac593aeaa8 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Sun, 17 Dec 2023 16:21:15 +0100 Subject: [PATCH 01/19] changelog github integration graphql query works mostly bearer not working anymore --- changed.d | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 95 insertions(+), 6 deletions(-) diff --git a/changed.d b/changed.d index 3b8d61f5d..78e6f1336 100755 --- a/changed.d +++ b/changed.d @@ -42,7 +42,7 @@ module changed; import std.net.curl, std.conv, std.exception, std.algorithm, std.csv, std.typecons, std.stdio, std.datetime, std.array, std.string, std.file, std.format, std.getopt, - std.path, std.functional; + std.path, std.functional, std.json; import std.range.primitives, std.traits; @@ -50,6 +50,7 @@ struct BugzillaEntry { int id; string summary; + Nullable!int githubId; } struct ChangelogEntry @@ -126,7 +127,7 @@ string escapeParens(string input) } /** Get a list of all bugzilla issues mentioned in revRange */ -auto getIssues(string revRange) +int[] getIssues(string revRange) { import std.process : execute, pipeProcess, Redirect, wait; import std.regex : ctRegex, match, splitter; @@ -155,6 +156,7 @@ auto getIssues(string revRange) foreach (line; p.stdout.byLine()) { + writeln(line); if (auto m = match(line.stripLeft, closedRE)) { m.captures[1] @@ -165,16 +167,22 @@ auto getIssues(string revRange) } } } - return issues.data.sort().release.uniq; + return issues.data.sort().release.uniq.array; +} + +BugzillaEntry[][string][string] getClosedGithubIssues(string revRange) +{ + BugzillaEntry[][string][string] entries; + return entries; } /** Generate and return the change log as a string. */ -auto getBugzillaChanges(string revRange) +BugzillaEntry[][string][string] getBugzillaChanges(string revRange) { // component (e.g. DMD) -> bug type (e.g. regression) -> list of bug entries BugzillaEntry[][string][string] entries; - auto issues = getIssues(revRange); + int[] issues = getIssues(revRange); // abort prematurely if no issues are found in all git logs if (issues.empty) return entries; @@ -224,6 +232,77 @@ auto getBugzillaChanges(string revRange) return entries; } +struct GithubIssue { + int number; + string title; + string body_; + DateTime closedAt; +} + +GithubIssue[] getGithubIssues(string repo, DateTime startDate, + DateTime endDate) +{ + const query = ` +query($repo: String!, $startDate: DateTime!) { + repository(owner: "burner", name: $repo) { + name + id + issues(states: CLOSED, first: 100 + , filterBy: { since: $startDate }) + { + edges { + node { + number + title + id + body + closedAt + } + } + pageInfo { + endCursor + startCursor + hasNextPage + hasPreviousPage + } + } + } +}`; + JSONValue toSend; + toSend["query"] = query; + toSend["variables"] = `{ "repo": "%s", "startDate": "%sZ", "endDate": "%sZ" }` + .format("graphqld", startDate.toISOExtString(), endDate.toISOExtString()); + string requestData = toSend.toPrettyString(); + + writefln("RD %s", requestData); + string ghToken = readText("gh_token").strip(); + string bearer = format("Bearer %s", ghToken); + writefln("'%s' '%s'", ghToken, bearer); + + HTTP http = HTTP("https://api.github.com/graphql"); + http.addRequestHeader("Authorization", bearer); + http.setPostData(requestData, "application/json"); + + char[] response; + try { + http.onReceive = (ubyte[] d) { + response = cast(char[])d; + return d.length; + }; + http.perform(); + } catch(Exception e) { + throw e; + } + + writefln("RS %s", cast(string)response); + + string s = cast(string)response; + JSONValue j = parseJSON(s); + writeln(j.toPrettyString()); + + return []; +} + /** Reads a single changelog file. @@ -391,6 +470,12 @@ void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) } } +int main(string[] args) { + getGithubIssues("phobos", DateTime(2023,1,1), DateTime(2023,12,1)); + return 0; +} + +__EOF__ int main(string[] args) { auto outputFile = "./changelog.dd"; @@ -434,6 +519,8 @@ Please supply a bugzilla version writeln("Skipped querying Bugzilla for changes. Please define a revision range e.g ./changed v2.072.2..upstream/stable"); } + getGithubIssues("phobos", DateTime(2023,1,1), DateTime(2023,12,1)); + // location of the changelog files alias Repo = Tuple!(string, "name", string, "headline", string, "path", string, "prefix"); auto repos = [Repo("dmd", "Compiler changes", "changelog", "dmd."), @@ -473,7 +560,9 @@ Please supply a bugzilla version // Accumulate Bugzilla issues typeof(revRange.getBugzillaChanges) bugzillaChanges; if (revRange.length) - bugzillaChanges = revRange.getBugzillaChanges; + { + bugzillaChanges = revRange.getBugzillaChanges(); + } // Accumulate contributors from the git log version(Contributors_Lib) From a34069cd43900aa43f44d14c1293521874ae4f64 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Sun, 7 Jan 2024 16:33:35 +0100 Subject: [PATCH 02/19] using rest to get the closed issues --- changed.d | 288 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 209 insertions(+), 79 deletions(-) diff --git a/changed.d b/changed.d index 78e6f1336..9dc50eab4 100755 --- a/changed.d +++ b/changed.d @@ -48,7 +48,7 @@ import std.range.primitives, std.traits; struct BugzillaEntry { - int id; + Nullable!int id; string summary; Nullable!int githubId; } @@ -126,8 +126,13 @@ string escapeParens(string input) return input.translate(parenToMacro); } +struct GitIssues { + int[] bugzillaIssueIds; + int[] githubIssueIds; +} + /** Get a list of all bugzilla issues mentioned in revRange */ -int[] getIssues(string revRange) +GitIssues getIssues(string revRange) { import std.process : execute, pipeProcess, Redirect, wait; import std.regex : ctRegex, match, splitter; @@ -139,9 +144,11 @@ int[] getIssues(string revRange) // issues reference that won't close the issue). // Note: "Bugzilla" is required since https://github.com/dlang/dlang-bot/pull/302; // temporarily both are accepted during a transition period. - enum closedRE = ctRegex!(`(?:^fix(?:es)?(?:\s+bugzilla)?(?:\s+(?:issues?|bugs?))?\s+(#?\d+(?:[\s,\+&and]+#?\d+)*))`, "i"); + enum closedREBZ = ctRegex!(`(?:^fix(?:es)?(?:\s+bugzilla)?(?:\s+(?:issues?|bugs?))?\s+(#?\d+(?:[\s,\+&and]+#?\d+)*))`, "i"); + enum closedREGH = ctRegex!(`(?:^fix(?:es)?(?:\s+github)?(?:\s+(?:issues?|bugs?))?\s+(#?\d+(?:[\s,\+&and]+#?\d+)*))`, "i"); - auto issues = appender!(int[]); + auto issuesBZ = appender!(int[]); + auto issuesGH = appender!(int[]); foreach (repo; ["dmd", "phobos", "dlang.org", "tools", "installer"] .map!(r => buildPath("..", r))) { @@ -157,37 +164,42 @@ int[] getIssues(string revRange) foreach (line; p.stdout.byLine()) { writeln(line); - if (auto m = match(line.stripLeft, closedRE)) + if (auto m = match(line.stripLeft, closedREBZ)) { m.captures[1] .splitter(ctRegex!`[^\d]+`) .filter!(b => b.length) .map!(to!int) - .copy(issues); + .copy(issuesBZ); + } + if (auto m = match(line.stripLeft, closedREGH)) + { + m.captures[1] + .splitter(ctRegex!`[^\d]+`) + .filter!(b => b.length) + .map!(to!int) + .copy(issuesGH); } } } - return issues.data.sort().release.uniq.array; -} - -BugzillaEntry[][string][string] getClosedGithubIssues(string revRange) -{ - BugzillaEntry[][string][string] entries; - return entries; + return GitIssues(issuesBZ.data.sort().release.uniq.array + ,issuesGH.data.sort().release.uniq.array); } /** Generate and return the change log as a string. */ -BugzillaEntry[][string][string] getBugzillaChanges(string revRange) +BugzillaEntry[][string /*type */][string /*comp*/] getBugzillaChanges(string revRange) { // component (e.g. DMD) -> bug type (e.g. regression) -> list of bug entries BugzillaEntry[][string][string] entries; - int[] issues = getIssues(revRange); + GitIssues issues = getIssues(revRange); // abort prematurely if no issues are found in all git logs - if (issues.empty) + if (issues.bugzillaIssueIds.empty) + { return entries; + } - auto req = generateRequest(templateRequest, issues); + auto req = generateRequest(templateRequest, issues.bugzillaIssueIds); debug stderr.writeln(req); // write text auto data = req.get; @@ -225,7 +237,7 @@ BugzillaEntry[][string][string] getBugzillaChanges(string revRange) default: assert(0, type); } - auto entry = BugzillaEntry(fields[0], fields[3].idup); + auto entry = BugzillaEntry(fields[0].nullable(), fields[3].idup); entries[comp][type] ~= entry; changelogStats.addBugzillaIssue(entry, comp, type); } @@ -236,71 +248,180 @@ struct GithubIssue { int number; string title; string body_; + string type; DateTime closedAt; + Nullable!int bugzillaId; +} + +Nullable!int getBugzillaId(string body_) { + string prefix = "### Transfered from https://issues.dlang.org/show_bug.cgi?id="; + ptrdiff_t pIdx = body_.indexOf(prefix); + Nullable!int ret; + if(pIdx != -1) { + ptrdiff_t newLine = body_.indexOf("\n", pIdx + prefix.length); + if(newLine != -1) { + ret = body_[pIdx + prefix.length .. newLine].to!int(); + } + } + return ret; } -GithubIssue[] getGithubIssues(string repo, DateTime startDate, - DateTime endDate) +GithubIssue[][string /*type*/ ][string /*comp*/] getGithubIssuesRest(const DateTime startDate, + const DateTime endDate, const string bearer) { - const query = ` -query($repo: String!, $startDate: DateTime!) { - repository(owner: "burner", name: $repo) { - name - id - issues(states: CLOSED, first: 100 - , filterBy: { since: $startDate }) - { - edges { - node { - number - title - id - body - closedAt - } - } - pageInfo { - endCursor - startCursor - hasNextPage - hasPreviousPage + GithubIssue[][string][string] ret; + string[2][] comps = + [ [ "dlang.org", "dlang.org"] + , [ "dmd", "DMD Compiler"] + , [ "druntime", "Druntime"] + , [ "phobos", "Phobos"] + , [ "tools", "Tools"] + , [ "dub", "Dub"] + , [ "visuald", "VisualD"] + ]; + foreach(it; comps) { + GithubIssue[][string /* type */] tmp; + GithubIssue[] ghi = getGithubIssuesRest("dlang", it[0], startDate, + endDate, bearer); + foreach(jt; ghi) { + GithubIssue[]* p = jt.type in tmp; + if(p !is null) { + (*p) ~= jt; + } else { + tmp[jt.type] = [jt]; } } - } -}`; - JSONValue toSend; - toSend["query"] = query; - toSend["variables"] = `{ "repo": "%s", "startDate": "%sZ", "endDate": "%sZ" }` - .format("graphqld", startDate.toISOExtString(), endDate.toISOExtString()); - string requestData = toSend.toPrettyString(); - - writefln("RD %s", requestData); - string ghToken = readText("gh_token").strip(); - string bearer = format("Bearer %s", ghToken); - writefln("'%s' '%s'", ghToken, bearer); - - HTTP http = HTTP("https://api.github.com/graphql"); - http.addRequestHeader("Authorization", bearer); - http.setPostData(requestData, "application/json"); - - char[] response; - try { - http.onReceive = (ubyte[] d) { - response = cast(char[])d; - return d.length; - }; - http.perform(); - } catch(Exception e) { - throw e; + ret[it[1]] = tmp; } + return ret; +} - writefln("RS %s", cast(string)response); +/** +Get closed issues of a github project - string s = cast(string)response; - JSONValue j = parseJSON(s); - writeln(j.toPrettyString()); +Params: + project = almost always the dlang github project + repo = the name of the repo to get the closed issues for + endDate = the cutoff date for closed issues + bearer = the classic github bearer token +*/ +GithubIssue[] getGithubIssuesRest(const string project, const string repo + , const DateTime startDate, const DateTime endDate, const string bearer) +{ + GithubIssue[] ret; + foreach(page; 1 .. 100) { // 1000 issues per release should be enough + string req = ("https://api.github.com/repos/%s/%s/issues?per_page=100" + ~"&state=closed&since=%s&page=%s") + .format(project, repo, endDate.toISOExtString() ~ "Z", page); + + writeln(req); + HTTP http = HTTP(req); + http.addRequestHeader("Accept", "application/vnd.github+json"); + http.addRequestHeader("X-GitHub-Api-Version", "2022-11-28"); + http.addRequestHeader("Authorization", bearer); + + char[] response; + try { + http.onReceive = (ubyte[] d) { + response ~= cast(char[])d; + return d.length; + }; + http.perform(); + } catch(Exception e) { + throw e; + } - return []; + string s = cast(string)response; + writeln(s); + JSONValue j = parseJSON(s); + enforce(j.type == JSONType.array, j.toPrettyString() + ~ "\nMust be an array"); + JSONValue[] arr = j.arrayNoRef(); + if(arr.empty) { + break; + } + foreach(it; arr) { + GithubIssue tmp; + { + JSONValue* mem = "number" in it; + enforce(mem !is null, it.toPrettyString() + ~ "\nmust contain 'number'"); + enforce((*mem).type == JSONType.integer, (*mem).toPrettyString() + ~ "\n'number' must be an integer"); + tmp.number = (*mem).get!int(); + } + { + JSONValue* mem = "title" in it; + enforce(mem !is null, it.toPrettyString() + ~ "\nmust contain 'title'"); + enforce((*mem).type == JSONType.string, (*mem).toPrettyString() + ~ "\n'title' must be an string"); + tmp.title = (*mem).get!string(); + } + { + JSONValue* mem = "body" in it; + enforce(mem !is null, it.toPrettyString() + ~ "\nmust contain 'body'"); + if((*mem).type == JSONType.string) { + tmp.body_ = (*mem).get!string(); + // get the possible bugzilla id + tmp.bugzillaId = getBugzillaId(tmp.body_); + } + } + { + JSONValue* mem = "closed_at" in it; + enforce(mem !is null, it.toPrettyString() + ~ "\nmust contain 'closed_at'"); + enforce((*mem).type == JSONType.string, (*mem).toPrettyString() + ~ "\n'closed_at' must be an string"); + string d = (*mem).get!string(); + d = d.endsWith("Z") + ? d[0 .. $ - 1] + : d; + tmp.closedAt = DateTime.fromISOExtString(d); + } + { + JSONValue* mem = "labels" in it; + tmp.type = "bug fixes"; + if(mem !is null) { + enforce(mem !is null, it.toPrettyString() + ~ "\nmust contain 'labels'"); + enforce((*mem).type == JSONType.array, (*mem).toPrettyString() + ~ "\n'labels' must be an string"); + foreach(l; (*mem).arrayNoRef()) { + enforce(l.type == JSONType.object, l.toPrettyString() + ~ "\nmust be an object"); + JSONValue* lbl = "name" in l; + enforce(lbl !is null, it.toPrettyString() + ~ "\nmust contain 'name'"); + enforce((*lbl).type == JSONType.string, (*lbl).toPrettyString() + ~ "\n'name' must be an string"); + string n = (*lbl).get!string(); + switch (n) + { + case "regression": + tmp.type = "regression fixes"; + break; + + case "blocker", "critical", "major", "normal", "minor", "trivial": + tmp.type = "bug fixes"; + break; + + case "enhancement": + tmp.type = "enhancements"; + break; + default: + } + } + } + } + ret ~= tmp; + } + if(arr.length < 100) { + break; + } + } + return ret; } /** @@ -438,6 +559,13 @@ void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string hea } } +bool less(ref BugzillaEntry a, ref BugzillaEntry b) { + if(!a.id.isNull() && !b.id.isNull()) { + return a.id.get() < b.id.get(); + } + return false; +} + /** Writes the fixed issued from Bugzilla in the ddoc format as a single list. @@ -459,7 +587,7 @@ void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) if (auto bugs = bugtype in *comp) { w.formattedWrite("$(BUGSTITLE_BUGZILLA %s %s,\n\n", component, bugtype); - foreach (bug; sort!"a.id < b.id"(*bugs)) + foreach (bug; sort!less(*bugs)) { w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n", bug.id, bug.summary.escapeParens()); @@ -470,12 +598,15 @@ void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) } } +/* int main(string[] args) { - getGithubIssues("phobos", DateTime(2023,1,1), DateTime(2023,12,1)); + GithubIssue[][string][string] ghIssues = getGithubIssuesRest(DateTime(2023,1,1), DateTime(2023,12,1) + , readText("gh_token").strip()); + writefln("%s", ghIssues); return 0; } +*/ -__EOF__ int main(string[] args) { auto outputFile = "./changelog.dd"; @@ -519,8 +650,6 @@ Please supply a bugzilla version writeln("Skipped querying Bugzilla for changes. Please define a revision range e.g ./changed v2.072.2..upstream/stable"); } - getGithubIssues("phobos", DateTime(2023,1,1), DateTime(2023,12,1)); - // location of the changelog files alias Repo = Tuple!(string, "name", string, "headline", string, "path", string, "prefix"); auto repos = [Repo("dmd", "Compiler changes", "changelog", "dmd."), @@ -558,7 +687,8 @@ Please supply a bugzilla version w.put("$(CHANGELOG_NAV_INJECT)\n\n"); // Accumulate Bugzilla issues - typeof(revRange.getBugzillaChanges) bugzillaChanges; + //typeof(revRange.getBugzillaChanges) bugzillaChanges; + BugzillaEntry[][string][string] bugzillaChanges; if (revRange.length) { bugzillaChanges = revRange.getBugzillaChanges(); From afb16361ad34e7280997be915ef5e3c7a4f75105 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Sun, 7 Jan 2024 16:38:13 +0100 Subject: [PATCH 03/19] started work on the integration next thing to do is wire in the closed github issues even if there where not closed by a commit --- changed.d | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changed.d b/changed.d index 9dc50eab4..1dfec9d4e 100755 --- a/changed.d +++ b/changed.d @@ -619,12 +619,14 @@ int main(string[] args) string previousVersion = "Previous version"; bool hideTextChanges = false; string revRange; + string githubClassicTokenFileName; auto helpInformation = getopt( args, std.getopt.config.passThrough, "output|o", &outputFile, "date", &nextVersionDate, + "ghToken|t", &githubClassicTokenFileName, "version", &nextVersionString, "prev-version", &previousVersion, // this can automatically be detected "no-text", &hideTextChanges); From 762a6b678db1fef68d8233b4857ec31f2269f732 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Sun, 7 Jan 2024 22:22:19 +0100 Subject: [PATCH 04/19] got the first date and used that for getting the github issues. now I need to integrate the github issues into the output --- changed.d | 54 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/changed.d b/changed.d index 1dfec9d4e..7349c4845 100755 --- a/changed.d +++ b/changed.d @@ -42,7 +42,7 @@ module changed; import std.net.curl, std.conv, std.exception, std.algorithm, std.csv, std.typecons, std.stdio, std.datetime, std.array, std.string, std.file, std.format, std.getopt, - std.path, std.functional, std.json; + std.path, std.functional, std.json, std.process; import std.range.primitives, std.traits; @@ -126,6 +126,31 @@ string escapeParens(string input) return input.translate(parenToMacro); } +Nullable!DateTime getFirstDateTime(string revRange) { + DateTime[] all; + + foreach (repo; ["dmd", "phobos", "dlang.org", "tools", "installer"] + .map!(r => buildPath("..", r))) + { + auto cmd = ["git", "log", "--no-patch", "--no-notes" + , "--date=format-local:%Y-%m-%dT%H:%M:%S", "--pretty=%cd" + , revRange]; + auto p = pipeProcess(cmd, Redirect.stdout); + all ~= p.stdout.byLine() + .map!((char[] l) { + auto r = DateTime.fromISOExtString(l); + return r; + }) + .array; + } + + all.sort(); + + return all.empty + ? Nullable!(DateTime).init + : all.front.nullable; +} + struct GitIssues { int[] bugzillaIssueIds; int[] githubIssueIds; @@ -134,7 +159,6 @@ struct GitIssues { /** Get a list of all bugzilla issues mentioned in revRange */ GitIssues getIssues(string revRange) { - import std.process : execute, pipeProcess, Redirect, wait; import std.regex : ctRegex, match, splitter; // Keep in sync with the regex in dlang-bot: @@ -163,7 +187,6 @@ GitIssues getIssues(string revRange) foreach (line; p.stdout.byLine()) { - writeln(line); if (auto m = match(line.stripLeft, closedREBZ)) { m.captures[1] @@ -312,9 +335,8 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo foreach(page; 1 .. 100) { // 1000 issues per release should be enough string req = ("https://api.github.com/repos/%s/%s/issues?per_page=100" ~"&state=closed&since=%s&page=%s") - .format(project, repo, endDate.toISOExtString() ~ "Z", page); + .format(project, repo, startDate.toISOExtString() ~ "Z", page); - writeln(req); HTTP http = HTTP(req); http.addRequestHeader("Accept", "application/vnd.github+json"); http.addRequestHeader("X-GitHub-Api-Version", "2022-11-28"); @@ -332,7 +354,6 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo } string s = cast(string)response; - writeln(s); JSONValue j = parseJSON(s); enforce(j.type == JSONType.array, j.toPrettyString() ~ "\nMust be an array"); @@ -612,7 +633,7 @@ int main(string[] args) auto outputFile = "./changelog.dd"; auto nextVersionString = "LATEST"; - auto currDate = Clock.currTime(); + SysTime currDate = Clock.currTime(); auto nextVersionDate = "%s %02d, %04d" .format(currDate.month.to!string.capitalize, currDate.day, currDate.year); @@ -638,6 +659,10 @@ Please supply a bugzilla version ./changed.d "v2.071.2..upstream/stable"`.defaultGetoptPrinter(helpInformation.options); } + assert(exists(githubClassicTokenFileName), format("No file with name '%s' exists" + , githubClassicTokenFileName)); + const string githubToken = readText(githubClassicTokenFileName).strip(); + if (args.length >= 2) { revRange = args[1]; @@ -652,6 +677,7 @@ Please supply a bugzilla version writeln("Skipped querying Bugzilla for changes. Please define a revision range e.g ./changed v2.072.2..upstream/stable"); } + // location of the changelog files alias Repo = Tuple!(string, "name", string, "headline", string, "path", string, "prefix"); auto repos = [Repo("dmd", "Compiler changes", "changelog", "dmd."), @@ -691,10 +717,20 @@ Please supply a bugzilla version // Accumulate Bugzilla issues //typeof(revRange.getBugzillaChanges) bugzillaChanges; BugzillaEntry[][string][string] bugzillaChanges; - if (revRange.length) + if (revRange.length >= 0) + { + bugzillaChanges = getBugzillaChanges(revRange); + } + + Nullable!(DateTime) firstDate = getFirstDateTime(revRange); + enforce(!firstDate.isNull(), "Couldn't find a date from the revRange"); + GithubIssue[][string][string] githubChanges; + if (revRange.length >= 0) { - bugzillaChanges = revRange.getBugzillaChanges(); + githubChanges = getGithubIssuesRest(firstDate.get(), cast(DateTime)currDate + , githubToken); } + writeln(githubChanges); // Accumulate contributors from the git log version(Contributors_Lib) From 2230cbb7a6bc18583a6eb470476e9c46763ca75d Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Wed, 10 Jan 2024 22:51:55 +0100 Subject: [PATCH 05/19] compiles with output, need to test it next --- changed.d | 60 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/changed.d b/changed.d index 7349c4845..b0ced2aad 100755 --- a/changed.d +++ b/changed.d @@ -53,6 +53,25 @@ struct BugzillaEntry Nullable!int githubId; } +struct GithubIssue { + int number; + string title; + string body_; + string type; + DateTime closedAt; + Nullable!int bugzillaId; + + @property string summary() const + { + return this.title; + } + + @property int id() const + { + return this.number; + } +} + struct ChangelogEntry { string title; // the first line (can't contain links) @@ -267,15 +286,6 @@ BugzillaEntry[][string /*type */][string /*comp*/] getBugzillaChanges(string rev return entries; } -struct GithubIssue { - int number; - string title; - string body_; - string type; - DateTime closedAt; - Nullable!int bugzillaId; -} - Nullable!int getBugzillaId(string body_) { string prefix = "### Transfered from https://issues.dlang.org/show_bug.cgi?id="; ptrdiff_t pIdx = body_.indexOf(prefix); @@ -580,13 +590,21 @@ void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string hea } } -bool less(ref BugzillaEntry a, ref BugzillaEntry b) { +bool lessImpl(ref BugzillaEntry a, ref BugzillaEntry b) { if(!a.id.isNull() && !b.id.isNull()) { return a.id.get() < b.id.get(); } return false; } +bool lessImpl(ref GithubIssue a, ref GithubIssue b) { + return a.number < b.number; +} + +bool less(T)(ref T a, ref T b) { + return lessImpl(a, b); +} + /** Writes the fixed issued from Bugzilla in the ddoc format as a single list. @@ -605,15 +623,18 @@ void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) if (auto comp = component in entries) { foreach (bugtype; bugtypes) - if (auto bugs = bugtype in *comp) { - w.formattedWrite("$(BUGSTITLE_BUGZILLA %s %s,\n\n", component, bugtype); - foreach (bug; sort!less(*bugs)) + if (auto bugs = bugtype in *comp) { - w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n", - bug.id, bug.summary.escapeParens()); + w.formattedWrite("$(BUGSTITLE_BUGZILLA %s %s,\n\n", component, bugtype); + alias lessFunc = less!(ElementEncodingType!(typeof(*bugs))); + foreach (bug; sort!lessFunc(*bugs)) + { + w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n", + bug.id, bug.summary.escapeParens()); + } + w.put(")\n"); } - w.put(")\n"); } } } @@ -730,7 +751,6 @@ Please supply a bugzilla version githubChanges = getGithubIssuesRest(firstDate.get(), cast(DateTime)currDate , githubToken); } - writeln(githubChanges); // Accumulate contributors from the git log version(Contributors_Lib) @@ -804,7 +824,13 @@ Please supply a bugzilla version // print the entire changelog history if (revRange.length) + { bugzillaChanges.writeBugzillaChanges(w); + } + if (revRange.length) + { + githubChanges.writeBugzillaChanges(w); + } } version(Contributors_Lib) From b626073c032881721b464e11a41ebc257e685b43 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Wed, 21 Feb 2024 13:16:21 +0100 Subject: [PATCH 06/19] last pass --- changed.d | 87 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/changed.d b/changed.d index b0ced2aad..cb79c66c8 100755 --- a/changed.d +++ b/changed.d @@ -53,7 +53,8 @@ struct BugzillaEntry Nullable!int githubId; } -struct GithubIssue { +struct GithubIssue +{ int number; string title; string body_; @@ -145,7 +146,8 @@ string escapeParens(string input) return input.translate(parenToMacro); } -Nullable!DateTime getFirstDateTime(string revRange) { +Nullable!DateTime getFirstDateTime(string revRange) +{ DateTime[] all; foreach (repo; ["dmd", "phobos", "dlang.org", "tools", "installer"] @@ -170,7 +172,8 @@ Nullable!DateTime getFirstDateTime(string revRange) { : all.front.nullable; } -struct GitIssues { +struct GitIssues +{ int[] bugzillaIssueIds; int[] githubIssueIds; } @@ -286,13 +289,16 @@ BugzillaEntry[][string /*type */][string /*comp*/] getBugzillaChanges(string rev return entries; } -Nullable!int getBugzillaId(string body_) { +Nullable!int getBugzillaId(string body_) +{ string prefix = "### Transfered from https://issues.dlang.org/show_bug.cgi?id="; ptrdiff_t pIdx = body_.indexOf(prefix); Nullable!int ret; - if(pIdx != -1) { + if (pIdx != -1) + { ptrdiff_t newLine = body_.indexOf("\n", pIdx + prefix.length); - if(newLine != -1) { + if (newLine != -1) + { ret = body_[pIdx + prefix.length .. newLine].to!int(); } } @@ -312,15 +318,20 @@ GithubIssue[][string /*type*/ ][string /*comp*/] getGithubIssuesRest(const DateT , [ "dub", "Dub"] , [ "visuald", "VisualD"] ]; - foreach(it; comps) { + foreach (it; comps) + { GithubIssue[][string /* type */] tmp; GithubIssue[] ghi = getGithubIssuesRest("dlang", it[0], startDate, endDate, bearer); - foreach(jt; ghi) { + foreach (jt; ghi) + { GithubIssue[]* p = jt.type in tmp; - if(p !is null) { + if (p !is null) + { (*p) ~= jt; - } else { + } + else + { tmp[jt.type] = [jt]; } } @@ -342,7 +353,8 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo , const DateTime startDate, const DateTime endDate, const string bearer) { GithubIssue[] ret; - foreach(page; 1 .. 100) { // 1000 issues per release should be enough + foreach (page; 1 .. 100) + { // 1000 issues per release should be enough string req = ("https://api.github.com/repos/%s/%s/issues?per_page=100" ~"&state=closed&since=%s&page=%s") .format(project, repo, startDate.toISOExtString() ~ "Z", page); @@ -353,13 +365,17 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo http.addRequestHeader("Authorization", bearer); char[] response; - try { - http.onReceive = (ubyte[] d) { + try + { + http.onReceive = (ubyte[] d) + { response ~= cast(char[])d; return d.length; }; http.perform(); - } catch(Exception e) { + } + catch(Exception e) + { throw e; } @@ -368,10 +384,12 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo enforce(j.type == JSONType.array, j.toPrettyString() ~ "\nMust be an array"); JSONValue[] arr = j.arrayNoRef(); - if(arr.empty) { + if (arr.empty) + { break; } - foreach(it; arr) { + foreach (it; arr) + { GithubIssue tmp; { JSONValue* mem = "number" in it; @@ -393,7 +411,8 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo JSONValue* mem = "body" in it; enforce(mem !is null, it.toPrettyString() ~ "\nmust contain 'body'"); - if((*mem).type == JSONType.string) { + if ((*mem).type == JSONType.string) + { tmp.body_ = (*mem).get!string(); // get the possible bugzilla id tmp.bugzillaId = getBugzillaId(tmp.body_); @@ -414,12 +433,14 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo { JSONValue* mem = "labels" in it; tmp.type = "bug fixes"; - if(mem !is null) { + if (mem !is null) + { enforce(mem !is null, it.toPrettyString() ~ "\nmust contain 'labels'"); enforce((*mem).type == JSONType.array, (*mem).toPrettyString() ~ "\n'labels' must be an string"); - foreach(l; (*mem).arrayNoRef()) { + foreach (l; (*mem).arrayNoRef()) + { enforce(l.type == JSONType.object, l.toPrettyString() ~ "\nmust be an object"); JSONValue* lbl = "name" in l; @@ -448,7 +469,8 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo } ret ~= tmp; } - if(arr.length < 100) { + if (arr.length < 100) + { break; } } @@ -537,7 +559,7 @@ void writeTextChangesHeader(Entries, Writer)(Entries changes, Writer w, string h // write the overview titles w.formattedWrite("$(BUGSTITLE_TEXT_HEADER %s,\n\n", headline); scope(exit) w.put("\n)\n\n"); - foreach(change; changes) + foreach (change; changes) { w.formattedWrite("$(LI $(RELATIVE_LINK2 %s,%s))\n", change.basename, change.title); } @@ -554,7 +576,7 @@ void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string hea { w.formattedWrite("$(BUGSTITLE_TEXT_BODY %s,\n\n", headline); scope(exit) w.put("\n)\n\n"); - foreach(change; changes) + foreach (change; changes) { w.formattedWrite("$(LI $(LNAME2 %s,%s)\n", change.basename, change.title); w.formattedWrite("$(CHANGELOG_SOURCE_FILE %s, %s)\n", change.repo, change.filePath); @@ -590,18 +612,22 @@ void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string hea } } -bool lessImpl(ref BugzillaEntry a, ref BugzillaEntry b) { - if(!a.id.isNull() && !b.id.isNull()) { +bool lessImpl(ref BugzillaEntry a, ref BugzillaEntry b) +{ + if (!a.id.isNull() && !b.id.isNull()) + { return a.id.get() < b.id.get(); } return false; } -bool lessImpl(ref GithubIssue a, ref GithubIssue b) { +bool lessImpl(ref GithubIssue a, ref GithubIssue b) +{ return a.number < b.number; } -bool less(T)(ref T a, ref T b) { +bool less(T)(ref T a, ref T b) +{ return lessImpl(a, b); } @@ -640,15 +666,6 @@ void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) } } -/* -int main(string[] args) { - GithubIssue[][string][string] ghIssues = getGithubIssuesRest(DateTime(2023,1,1), DateTime(2023,12,1) - , readText("gh_token").strip()); - writefln("%s", ghIssues); - return 0; -} -*/ - int main(string[] args) { auto outputFile = "./changelog.dd"; From d69fc9aed75aa8da9b17e3791d116b6063daa098 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Sat, 24 Feb 2024 21:49:23 +0100 Subject: [PATCH 07/19] working on the last error --- changed.d | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/changed.d b/changed.d index cb79c66c8..586f17ea2 100755 --- a/changed.d +++ b/changed.d @@ -392,7 +392,7 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo { GithubIssue tmp; { - JSONValue* mem = "number" in it; + const(JSONValue)* mem = "number" in it; enforce(mem !is null, it.toPrettyString() ~ "\nmust contain 'number'"); enforce((*mem).type == JSONType.integer, (*mem).toPrettyString() @@ -400,7 +400,7 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo tmp.number = (*mem).get!int(); } { - JSONValue* mem = "title" in it; + const(JSONValue)* mem = "title" in it; enforce(mem !is null, it.toPrettyString() ~ "\nmust contain 'title'"); enforce((*mem).type == JSONType.string, (*mem).toPrettyString() @@ -408,7 +408,7 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo tmp.title = (*mem).get!string(); } { - JSONValue* mem = "body" in it; + const(JSONValue)* mem = "body" in it; enforce(mem !is null, it.toPrettyString() ~ "\nmust contain 'body'"); if ((*mem).type == JSONType.string) @@ -419,7 +419,7 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo } } { - JSONValue* mem = "closed_at" in it; + const(JSONValue)* mem = "closed_at" in it; enforce(mem !is null, it.toPrettyString() ~ "\nmust contain 'closed_at'"); enforce((*mem).type == JSONType.string, (*mem).toPrettyString() @@ -431,7 +431,7 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo tmp.closedAt = DateTime.fromISOExtString(d); } { - JSONValue* mem = "labels" in it; + const(JSONValue)* mem = "labels" in it; tmp.type = "bug fixes"; if (mem !is null) { @@ -443,7 +443,7 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo { enforce(l.type == JSONType.object, l.toPrettyString() ~ "\nmust be an object"); - JSONValue* lbl = "name" in l; + const(JSONValue)* lbl = "name" in l; enforce(lbl !is null, it.toPrettyString() ~ "\nmust contain 'name'"); enforce((*lbl).type == JSONType.string, (*lbl).toPrettyString() From 00bf95feeaad1a17aea315251e020b3de1623a7e Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Fri, 17 Jan 2025 17:16:06 +0100 Subject: [PATCH 08/19] writeBugzillaChanges switch --- changed.d | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/changed.d b/changed.d index 586f17ea2..e2aaad945 100755 --- a/changed.d +++ b/changed.d @@ -631,6 +631,11 @@ bool less(T)(ref T a, ref T b) return lessImpl(a, b); } +enum BugzillaOrGithub { + bugzilla, + github +} + /** Writes the fixed issued from Bugzilla in the ddoc format as a single list. @@ -638,11 +643,23 @@ Params: changes = parsed InputRange of changelog information w = Output range to use */ -void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) +void writeBugzillaChanges(Entries, Writer)(BugzillaOrGithub bog, Entries entries, Writer w) if (isOutputRange!(Writer, string)) { immutable components = ["DMD Compiler", "Phobos", "Druntime", "dlang.org", "Optlink", "Tools", "Installer"]; immutable bugtypes = ["regression fixes", "bug fixes", "enhancements"]; + const string macroTitle = () { + final switch(bog) { + case BugzillaOrGithub.bugzilla: return "BUGSTITLE_BUGZILLA"; + case BugzillaOrGithub.github: return "BUGSTITLE_GITHUB"; + } + }(); + const string macroLi = () { + final switch(bog) { + case BugzillaOrGithub.bugzilla: return "BUGZILLA"; + case BugzillaOrGithub.github: return "GITHUB"; + } + }(); foreach (component; components) { @@ -652,11 +669,11 @@ void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) { if (auto bugs = bugtype in *comp) { - w.formattedWrite("$(BUGSTITLE_BUGZILLA %s %s,\n\n", component, bugtype); + w.formattedWrite("$(%s %s %s,\n\n", macroTitle, component, bugtype); alias lessFunc = less!(ElementEncodingType!(typeof(*bugs))); foreach (bug; sort!lessFunc(*bugs)) { - w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n", + w.formattedWrite("$(LI $(%s %s): %s)\n", macroLi, bug.id, bug.summary.escapeParens()); } w.put(")\n"); @@ -842,11 +859,11 @@ Please supply a bugzilla version // print the entire changelog history if (revRange.length) { - bugzillaChanges.writeBugzillaChanges(w); + writeBugzillaChanges(BugzillaOrGithub.bugzilla, bugzillaChanges, w); } if (revRange.length) { - githubChanges.writeBugzillaChanges(w); + writeBugzillaChanges(BugzillaOrGithub.github, githubChanges, w); } } From bea251e88fb4cbbc6c1b59070c09c98e55dfa9b4 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Sat, 8 Feb 2025 14:43:08 +0100 Subject: [PATCH 09/19] skip PR's and wrongly imported issues --- changed.d | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/changed.d b/changed.d index e2aaad945..2958b0186 100755 --- a/changed.d +++ b/changed.d @@ -355,9 +355,21 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo GithubIssue[] ret; foreach (page; 1 .. 100) { // 1000 issues per release should be enough - string req = ("https://api.github.com/repos/%s/%s/issues?per_page=100" + string req = ("https://api.github.com/repos/%s/%s/issues?per_page=100&since=%s" ~"&state=closed&since=%s&page=%s") - .format(project, repo, startDate.toISOExtString() ~ "Z", page); + .format(project, repo, + () { + switch(repo) { + case "dmd": return "2024-12-01T12:00:00Z"; + case "phobos": return "2024-12-01T12:00:00Z"; + case "tools": return "2001-01-01T00:00:00Z"; + case "dub": return "2001-01-01T00:00:00Z"; + case "visuald": return "2023-10-18T00:00:00Z"; + case "installer": return "2001-01-01T00:00:00Z"; + default: return "2001-01-01T00:00:00Z"; + } + }() + , startDate.toISOExtString() ~ "Z", page); HTTP http = HTTP(req); http.addRequestHeader("Accept", "application/vnd.github+json"); @@ -391,6 +403,12 @@ GithubIssue[] getGithubIssuesRest(const string project, const string repo foreach (it; arr) { GithubIssue tmp; + // Issues and pull request are both returned by the github api + // the changelog only contains closed issues so we need to skip + // PRs + if("pull_request" in it && it["pull_request"].type != JSONType.null_) { + continue; + } { const(JSONValue)* mem = "number" in it; enforce(mem !is null, it.toPrettyString() From 4f0609a17915c0ed38f5e7d9539ebcaac68ac893 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Wed, 5 Mar 2025 22:32:59 +0100 Subject: [PATCH 10/19] Use indexOfAny to find line ending Newlines in issues could either be `\n` or `\r\n`. --- changed.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changed.d b/changed.d index 2958b0186..669ca2552 100755 --- a/changed.d +++ b/changed.d @@ -296,7 +296,7 @@ Nullable!int getBugzillaId(string body_) Nullable!int ret; if (pIdx != -1) { - ptrdiff_t newLine = body_.indexOf("\n", pIdx + prefix.length); + ptrdiff_t newLine = body_.indexOfAny("\n\r", pIdx + prefix.length); if (newLine != -1) { ret = body_[pIdx + prefix.length .. newLine].to!int(); From 64d5cd876f6710988d4a4f7eaa4986f52698471c Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Wed, 5 Mar 2025 22:46:44 +0100 Subject: [PATCH 11/19] Only extract date from revRange when set When no revision range is given, should instead skip all checking of bugzilla and gitlab. --- changed.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changed.d b/changed.d index 669ca2552..4c6c28055 100755 --- a/changed.d +++ b/changed.d @@ -795,11 +795,11 @@ Please supply a bugzilla version bugzillaChanges = getBugzillaChanges(revRange); } - Nullable!(DateTime) firstDate = getFirstDateTime(revRange); - enforce(!firstDate.isNull(), "Couldn't find a date from the revRange"); GithubIssue[][string][string] githubChanges; if (revRange.length >= 0) { + Nullable!(DateTime) firstDate = getFirstDateTime(revRange); + enforce(!firstDate.isNull(), "Couldn't find a date from the revRange"); githubChanges = getGithubIssuesRest(firstDate.get(), cast(DateTime)currDate , githubToken); } From 7eff0ae3c1855f0818ae985a7b370bf1bc79f45e Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Wed, 5 Mar 2025 22:58:26 +0100 Subject: [PATCH 12/19] Replace assert with warning message Prefer to skip checking GitHub when no access token is provided. --- changed.d | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/changed.d b/changed.d index 4c6c28055..79e3627cc 100755 --- a/changed.d +++ b/changed.d @@ -732,9 +732,11 @@ Please supply a bugzilla version ./changed.d "v2.071.2..upstream/stable"`.defaultGetoptPrinter(helpInformation.options); } - assert(exists(githubClassicTokenFileName), format("No file with name '%s' exists" - , githubClassicTokenFileName)); - const string githubToken = readText(githubClassicTokenFileName).strip(); + if (githubClassicTokenFileName.empty) + { + writeln("Skipped querying GitHub for changes. Please provide an access token e.g ./changed -t token-file"); + writeln("To create a new token, visit https://github.com/settings/tokens/new"); + } if (args.length >= 2) { From 079d4701966224f112ebe42409c4c352d84f6a83 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Wed, 5 Mar 2025 23:09:03 +0100 Subject: [PATCH 13/19] Only check for existence of GH token file if --ghToken provided Treat the absence of the option as skipping the check for GitHub issues. --- changed.d | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/changed.d b/changed.d index 79e3627cc..c43be2e5e 100755 --- a/changed.d +++ b/changed.d @@ -790,16 +790,19 @@ Please supply a bugzilla version w.put("$(CHANGELOG_NAV_INJECT)\n\n"); // Accumulate Bugzilla issues - //typeof(revRange.getBugzillaChanges) bugzillaChanges; BugzillaEntry[][string][string] bugzillaChanges; - if (revRange.length >= 0) + if (!revRange.empty) { bugzillaChanges = getBugzillaChanges(revRange); } GithubIssue[][string][string] githubChanges; - if (revRange.length >= 0) + if (!revRange.empty && !githubClassicTokenFileName.empty) { + enforce(exists(githubClassicTokenFileName), format("No file with name '%s' exists" + , githubClassicTokenFileName)); + const string githubToken = readText(githubClassicTokenFileName).strip(); + Nullable!(DateTime) firstDate = getFirstDateTime(revRange); enforce(!firstDate.isNull(), "Couldn't find a date from the revRange"); githubChanges = getGithubIssuesRest(firstDate.get(), cast(DateTime)currDate From 7b48e78165a7b08ff70d0514ea2cc678904bb3cd Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Thu, 6 Mar 2025 00:24:46 +0100 Subject: [PATCH 14/19] Add getEndDateTime helper --- changed.d | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/changed.d b/changed.d index c43be2e5e..587bf55f2 100755 --- a/changed.d +++ b/changed.d @@ -172,6 +172,28 @@ Nullable!DateTime getFirstDateTime(string revRange) : all.front.nullable; } +Nullable!DateTime getEndDateTime(string nextVersionDate) +{ + // Input format: "MMM DD, YYYY" + if (nextVersionDate.length != 12 || + nextVersionDate[3] != ' ' || + nextVersionDate[6] != ',' || nextVersionDate[7] != ' ') + { + return Nullable!(DateTime).init; + } + + // Converted format: YYYY-MMM-DD + string simpleString; + simpleString ~= nextVersionDate[8..12]; + simpleString ~= "-"; + simpleString ~= nextVersionDate[0..3]; + simpleString ~= "-"; + simpleString ~= nextVersionDate[4..6]; + auto endDate = Date.fromSimpleString(simpleString); + + return DateTime(endDate).nullable; +} + struct GitIssues { int[] bugzillaIssueIds; From aa730b89ffbb35da09e52613d9e93394f0d159ff Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Thu, 6 Mar 2025 00:30:05 +0100 Subject: [PATCH 15/19] Use nextVersionDate as end date when searching github issues Issues closed after the release date should be ignored. --- changed.d | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/changed.d b/changed.d index 587bf55f2..0e58c00b8 100755 --- a/changed.d +++ b/changed.d @@ -827,8 +827,10 @@ Please supply a bugzilla version Nullable!(DateTime) firstDate = getFirstDateTime(revRange); enforce(!firstDate.isNull(), "Couldn't find a date from the revRange"); - githubChanges = getGithubIssuesRest(firstDate.get(), cast(DateTime)currDate - , githubToken); + Nullable!(DateTime) endDate = getEndDateTime(nextVersionDate); + enforce(!endDate.isNull(), "Couldn't parse the next version date string"); + + githubChanges = getGithubIssuesRest(firstDate.get(), endDate.get(), githubToken); } // Accumulate contributors from the git log From baf84e490f18ee2c863bf5b43ea4bce386e7599f Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Thu, 6 Mar 2025 05:14:00 +0100 Subject: [PATCH 16/19] Revert aa730b8 Switch back to using current date, worst case for now is wait time for large response. --- changed.d | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/changed.d b/changed.d index 0e58c00b8..587bf55f2 100755 --- a/changed.d +++ b/changed.d @@ -827,10 +827,8 @@ Please supply a bugzilla version Nullable!(DateTime) firstDate = getFirstDateTime(revRange); enforce(!firstDate.isNull(), "Couldn't find a date from the revRange"); - Nullable!(DateTime) endDate = getEndDateTime(nextVersionDate); - enforce(!endDate.isNull(), "Couldn't parse the next version date string"); - - githubChanges = getGithubIssuesRest(firstDate.get(), endDate.get(), githubToken); + githubChanges = getGithubIssuesRest(firstDate.get(), cast(DateTime)currDate + , githubToken); } // Accumulate contributors from the git log From 09d0a584206bf9c1b8397132d256798516742873 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Thu, 6 Mar 2025 05:14:31 +0100 Subject: [PATCH 17/19] Revert 7b48e78 Now unused. --- changed.d | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/changed.d b/changed.d index 587bf55f2..c43be2e5e 100755 --- a/changed.d +++ b/changed.d @@ -172,28 +172,6 @@ Nullable!DateTime getFirstDateTime(string revRange) : all.front.nullable; } -Nullable!DateTime getEndDateTime(string nextVersionDate) -{ - // Input format: "MMM DD, YYYY" - if (nextVersionDate.length != 12 || - nextVersionDate[3] != ' ' || - nextVersionDate[6] != ',' || nextVersionDate[7] != ' ') - { - return Nullable!(DateTime).init; - } - - // Converted format: YYYY-MMM-DD - string simpleString; - simpleString ~= nextVersionDate[8..12]; - simpleString ~= "-"; - simpleString ~= nextVersionDate[0..3]; - simpleString ~= "-"; - simpleString ~= nextVersionDate[4..6]; - auto endDate = Date.fromSimpleString(simpleString); - - return DateTime(endDate).nullable; -} - struct GitIssues { int[] bugzillaIssueIds; From 773b50ed1fcd96844a8ce57e34ba35855e376f13 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Thu, 6 Mar 2025 05:29:21 +0100 Subject: [PATCH 18/19] Filter out closed issues that are unfound in git logs Issues may be closed by a PR to the master branch, they should not appear as a changelog entry against stable. --- changed.d | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/changed.d b/changed.d index c43be2e5e..29d6e56a5 100755 --- a/changed.d +++ b/changed.d @@ -175,7 +175,7 @@ Nullable!DateTime getFirstDateTime(string revRange) struct GitIssues { int[] bugzillaIssueIds; - int[] githubIssueIds; + int[][string] githubIssueIds; } /** Get a list of all bugzilla issues mentioned in revRange */ @@ -194,7 +194,7 @@ GitIssues getIssues(string revRange) enum closedREGH = ctRegex!(`(?:^fix(?:es)?(?:\s+github)?(?:\s+(?:issues?|bugs?))?\s+(#?\d+(?:[\s,\+&and]+#?\d+)*))`, "i"); auto issuesBZ = appender!(int[]); - auto issuesGH = appender!(int[]); + int[][string] issuesGH; foreach (repo; ["dmd", "phobos", "dlang.org", "tools", "installer"] .map!(r => buildPath("..", r))) { @@ -207,6 +207,7 @@ GitIssues getIssues(string revRange) p = pipeProcess(cmd, Redirect.stdout); scope(exit) enforce(wait(p.pid) == 0, "Failed to execute '%(%s %)'.".format(cmd)); + auto matchesGH = appender!(int[]); foreach (line; p.stdout.byLine()) { if (auto m = match(line.stripLeft, closedREBZ)) @@ -223,12 +224,12 @@ GitIssues getIssues(string revRange) .splitter(ctRegex!`[^\d]+`) .filter!(b => b.length) .map!(to!int) - .copy(issuesGH); + .copy(matchesGH); } } + issuesGH[repo[3..$]] = matchesGH.data.sort().release.uniq.array; } - return GitIssues(issuesBZ.data.sort().release.uniq.array - ,issuesGH.data.sort().release.uniq.array); + return GitIssues(issuesBZ.data.sort().release.uniq.array, issuesGH); } /** Generate and return the change log as a string. */ @@ -305,26 +306,39 @@ Nullable!int getBugzillaId(string body_) return ret; } -GithubIssue[][string /*type*/ ][string /*comp*/] getGithubIssuesRest(const DateTime startDate, - const DateTime endDate, const string bearer) +GithubIssue[][string /*type*/ ][string /*comp*/] getGithubIssuesRest(string revRange, + const DateTime startDate, const DateTime endDate, const string bearer) { + import std.algorithm.searching : canFind; + GithubIssue[][string][string] ret; string[2][] comps = [ [ "dlang.org", "dlang.org"] , [ "dmd", "DMD Compiler"] - , [ "druntime", "Druntime"] + //, [ "druntime", "Druntime"] // Archived , [ "phobos", "Phobos"] , [ "tools", "Tools"] - , [ "dub", "Dub"] - , [ "visuald", "VisualD"] + //, [ "dub", "Dub"] // ???: Not searched in getIssues + //, [ "visuald", "VisualD"] // ???: + , [ "installer", "Installer"] ]; + GitIssues issues = getIssues(revRange); foreach (it; comps) { + // abort prematurely if no issues are found in all git logs + string project = it[0]; + if (project !in issues.githubIssueIds || issues.githubIssueIds[project].empty) + continue; + GithubIssue[][string /* type */] tmp; - GithubIssue[] ghi = getGithubIssuesRest("dlang", it[0], startDate, + GithubIssue[] ghi = getGithubIssuesRest("dlang", project, startDate, endDate, bearer); foreach (jt; ghi) { + // ignore if closed issue does not have a git log reference + if (!issues.githubIssueIds[project].canFind(jt.id)) + continue; + GithubIssue[]* p = jt.type in tmp; if (p !is null) { @@ -805,7 +819,7 @@ Please supply a bugzilla version Nullable!(DateTime) firstDate = getFirstDateTime(revRange); enforce(!firstDate.isNull(), "Couldn't find a date from the revRange"); - githubChanges = getGithubIssuesRest(firstDate.get(), cast(DateTime)currDate + githubChanges = getGithubIssuesRest(revRange, firstDate.get(), cast(DateTime)currDate , githubToken); } From f0e0cd2502bcc7b7b2110d85ca007a1e8531f168 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Thu, 6 Mar 2025 05:48:32 +0100 Subject: [PATCH 19/19] Prefix github issue macros with project name This is because each project has a different url path to link to their issues. --- changed.d | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/changed.d b/changed.d index 29d6e56a5..fdbb7c894 100755 --- a/changed.d +++ b/changed.d @@ -312,6 +312,7 @@ GithubIssue[][string /*type*/ ][string /*comp*/] getGithubIssuesRest(string revR import std.algorithm.searching : canFind; GithubIssue[][string][string] ret; + // Keep this list of comps in sync with the switch statement in writeBugzillaChanges string[2][] comps = [ [ "dlang.org", "dlang.org"] , [ "dmd", "DMD Compiler"] @@ -686,12 +687,21 @@ void writeBugzillaChanges(Entries, Writer)(BugzillaOrGithub bog, Entries entries case BugzillaOrGithub.github: return "BUGSTITLE_GITHUB"; } }(); - const string macroLi = () { + const macroLi = (string component) { final switch(bog) { - case BugzillaOrGithub.bugzilla: return "BUGZILLA"; - case BugzillaOrGithub.github: return "GITHUB"; + case BugzillaOrGithub.bugzilla: + return "BUGZILLA"; + case BugzillaOrGithub.github: + final switch (component) + { + case "dlang.org": return "DLANGORGGITHUB"; + case "DMD Compiler": return "DMDGITHUB"; + case "Phobos": return "PHOBOSGITHUB"; + case "Tools": return "TOOLSGITHUB"; + case "Installer": return "INSTALLERGITHUB"; + } } - }(); + }; foreach (component; components) { @@ -705,7 +715,7 @@ void writeBugzillaChanges(Entries, Writer)(BugzillaOrGithub bog, Entries entries alias lessFunc = less!(ElementEncodingType!(typeof(*bugs))); foreach (bug; sort!lessFunc(*bugs)) { - w.formattedWrite("$(LI $(%s %s): %s)\n", macroLi, + w.formattedWrite("$(LI $(%s %s): %s)\n", macroLi(component), bug.id, bug.summary.escapeParens()); } w.put(")\n");