feat(build-cli): New vnext generate:changelog command (#24899)

Adds a replacement vnext command for `generate:changelog`. The new
command is based on the BuildProject infrastructure as all vnext
commands are.

The primary change in this command is that git is no longer used to
revert changes. All changes are just made locally and must be committed
manually. The main thing we were reverting was a version bump, which is
trivial to reset using the `setVersion` function in build-infra.

I also moved the `canonicalizeChangeset` function to a shared location
and call it from both the old and new commands.

Once we feel good about the new command, we can remove the old one,
replace it with the one in vnext, and add a deprecated alias for the
now-removed vnext command.

A lot of the approach here is informed by
changesets/changesets#595, which is basically
what we'd like. Ideally we could ask changesets to generate changelogs
for a version that we provide (without also bumping versions), but that
doesn't exist at the moment.

Author: tylerbutler

build-tools/packages/build-cli/docs/vnext.md

@@ -4,19 +4,20 @@
 Vnext commands are new implementations of standard flub commands using new infrastructure.
 
 * [`flub vnext check latestVersions`](#flub-vnext-check-latestversions)
+* [`flub vnext generate changelog`](#flub-vnext-generate-changelog)
 
 ## `flub vnext check latestVersions`
 
 Determines if an input version matches a latest minor release version. Intended to be used in the Fluid Framework CI pipeline only.
 
 ```
 USAGE
-  $ flub vnext check latestVersions --releaseGroup <value> --version <value> [-v | --quiet]
+  $ flub vnext check latestVersions -g <value> --version <value> [-v | --quiet]
 
 FLAGS
-  --releaseGroup=<value>  (required) The name of a release group.
-  --version=<value>       (required) The version to check. When running in CI, this value corresponds to the pipeline
-                          trigger branch.
+  -g, --releaseGroup=<value>  (required) The name of a release group.
+      --version=<value>       (required) The version to check. When running in CI, this value corresponds to the
+                              pipeline trigger branch.
 
 LOGGING FLAGS
   -v, --verbose  Enable verbose logging.
@@ -31,3 +32,34 @@ DESCRIPTION
 ```
 
 _See code: [src/commands/vnext/check/latestVersions.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/vnext/check/latestVersions.ts)_
+
+## `flub vnext generate changelog`
+
+Generate a changelog for packages based on changesets. Note that this process deletes the changeset files!
+
+```
+USAGE
+  $ flub vnext generate changelog -g <value> [-v | --quiet] [--version <value>]
+
+FLAGS
+  -g, --releaseGroup=<value>  (required) The name of a release group.
+      --version=<value>       The version for which to generate the changelog. If this is not provided, the version of
+                              the package according to package.json will be used.
+
+LOGGING FLAGS
+  -v, --verbose  Enable verbose logging.
+      --quiet    Disable all logging.
+
+DESCRIPTION
+  Generate a changelog for packages based on changesets. Note that this process deletes the changeset files!
+
+ALIASES
+  $ flub vnext generate changelogs
+
+EXAMPLES
+  Generate changelogs for the client release group.
+
+    $ flub vnext generate changelog --releaseGroup client
+```
+
+_See code: [src/commands/vnext/generate/changelog.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/vnext/generate/changelog.ts)_

build-tools/packages/build-cli/src/commands/generate/changelog.ts

@@ -4,7 +4,6 @@
  */
 
 import { readFile, writeFile } from "node:fs/promises";
-import path from "node:path";
 import {
 	type VersionBumpType,
 	bumpVersionScheme,
@@ -17,7 +16,9 @@ import { inc } from "semver";
 import { CleanOptions } from "simple-git";
 
 import { checkFlags, releaseGroupFlag, semverFlag } from "../../flags.js";
-import { BaseCommand, DEFAULT_CHANGESET_PATH, loadChangesets } from "../../library/index.js";
+// eslint-disable-next-line import/no-internal-modules
+import { canonicalizeChangesets } from "../../library/changesets.js";
+import { BaseCommand } from "../../library/index.js";
 import { isReleaseGroup } from "../../releaseGroups.js";
 
 async function replaceInFile(
@@ -54,11 +55,8 @@ export default class GenerateChangeLogCommand extends BaseCommand<
 		},
 	];
 
-	private bumpType?: VersionBumpType;
-
-	private async processPackage(pkg: Package): Promise<void> {
+	private async processPackage(pkg: Package, bumpType: VersionBumpType): Promise<void> {
 		const { directory, version: pkgVersion } = pkg;
-		const bumpType = this.bumpType ?? "patch";
 
 		// This is the version that the changesets tooling calculates by default. It does a bump of the highest semver type
 		// in the changesets on the current version. We search for that version in the generated changelog and replace it
@@ -83,49 +81,6 @@ export default class GenerateChangeLogCommand extends BaseCommand<
 		);
 	}
 
-	/**
-	 * Removes any custom metadata from all changesets and writes the resulting changes back to the source files. This
-	 * metadata needs to be removed prior to running `changeset version` from the \@changesets/cli package. If it is not,
-	 * then the custom metadata is interpreted as change metadata and the changeset tools fail.
-	 *
-	 * For more information about the custom metadata we use in our changesets, see
-	 * https://github.com/microsoft/FluidFramework/wiki/Changesets#custom-metadata
-	 *
-	 * **Note that this is a lossy action!** The metadata is completely removed. Changesets are typically in source
-	 * control so changes can usually be reverted.
-	 */
-	private async canonicalizeChangesets(releaseGroupRootDir: string): Promise<void> {
-		const changesetDir = path.join(releaseGroupRootDir, DEFAULT_CHANGESET_PATH);
-		const changesets = await loadChangesets(changesetDir, this.logger);
-
-		// Determine the highest bump type and save it for later - it determines the changesets-calculated version.
-		const bumpTypes: Set<VersionBumpType> = new Set();
-		for (const changeset of changesets) {
-			for (const bumpType of changeset.changeTypes) {
-				bumpTypes.add(bumpType);
-			}
-		}
-		this.bumpType = bumpTypes.has("major")
-			? "major"
-			: bumpTypes.has("minor")
-				? "minor"
-				: "patch";
-
-		const toWrite: Promise<void>[] = [];
-		for (const changeset of changesets) {
-			// Filter out properties that start with __
-			const metadata = Object.entries(changeset.metadata)
-				.filter(([key]) => !key.startsWith("__"))
-				.map(([packageName, bump]) => {
-					return `"${packageName}": ${bump}`;
-				});
-			const output = `---\n${metadata.join("\n")}\n---\n\n${changeset.summary}\n\n${changeset.body}\n`;
-			this.info(`Writing canonical changeset: ${changeset.sourceFile}`);
-			toWrite.push(writeFile(changeset.sourceFile, output));
-		}
-		await Promise.all(toWrite);
-	}
-
 	public async run(): Promise<void> {
 		const context = await this.getContext();
 
@@ -147,7 +102,7 @@ export default class GenerateChangeLogCommand extends BaseCommand<
 
 		// Strips additional custom metadata from the source files before we call `changeset version`,
 		// because the changeset tools - like @changesets/cli - only work on canonical changesets.
-		await this.canonicalizeChangesets(releaseGroupRoot);
+		const bumpType = await canonicalizeChangesets(releaseGroupRoot, this.logger);
 
 		// The `changeset version` command applies the changesets to the changelogs
 		ux.action.start("Running `changeset version`");
@@ -179,7 +134,7 @@ export default class GenerateChangeLogCommand extends BaseCommand<
 		ux.action.start("Processing changelog updates");
 		const processPromises: Promise<void>[] = [];
 		for (const pkg of packagesToCheck) {
-			processPromises.push(this.processPackage(pkg));
+			processPromises.push(this.processPackage(pkg, bumpType));
 		}
 		const results = await Promise.allSettled(processPromises);
 		const failures = results.filter((p) => p.status === "rejected");

build-tools/packages/build-cli/src/commands/vnext/generate/changelog.ts

@@ -0,0 +1,104 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ux } from "@oclif/core";
+import { command as execCommand } from "execa";
+import { parse } from "semver";
+
+import { setVersion } from "@fluid-tools/build-infrastructure";
+import { releaseGroupNameFlag, semverFlag } from "../../../flags.js";
+// eslint-disable-next-line import/no-internal-modules
+import { updateChangelogs } from "../../../library/changelogs.js";
+// eslint-disable-next-line import/no-internal-modules
+import { canonicalizeChangesets } from "../../../library/changesets.js";
+import { BaseCommandWithBuildProject } from "../../../library/index.js";
+
+/**
+ * Generate a changelog for packages based on changesets. Note that this process deletes the changeset files!
+ *
+ * The reason we use a search/replace approach to update the version strings in the changelogs is largely because of
+ * https://github.com/changesets/changesets/issues/595. What we would like to do is generate the changelogs without
+ * doing version bumping, but that feature does not exist in the changeset tools.
+ */
+export default class GenerateChangeLogCommand extends BaseCommandWithBuildProject<
+	typeof GenerateChangeLogCommand
+> {
+	static readonly description =
+		"Generate a changelog for packages based on changesets. Note that this process deletes the changeset files!";
+
+	static readonly aliases = ["vnext:generate:changelogs"];
+
+	static readonly flags = {
+		releaseGroup: releaseGroupNameFlag({ required: true }),
+		version: semverFlag({
+			description:
+				"The version for which to generate the changelog. If this is not provided, the version of the package according to package.json will be used.",
+		}),
+		...BaseCommandWithBuildProject.flags,
+	} as const;
+
+	static readonly examples = [
+		{
+			description: "Generate changelogs for the client release group.",
+			command: "<%= config.bin %> <%= command.id %> --releaseGroup client",
+		},
+	];
+
+	public async run(): Promise<void> {
+		const buildProject = this.getBuildProject();
+
+		const { releaseGroup: releaseGroupName, version: versionOverride } = this.flags;
+
+		const releaseGroup = buildProject.releaseGroups.get(releaseGroupName);
+		if (releaseGroup === undefined) {
+			this.error(`Can't find release group named '${releaseGroupName}'`, { exit: 1 });
+		}
+
+		const releaseGroupRoot = releaseGroup.workspace.directory;
+		const releaseGroupVersion = parse(releaseGroup.version);
+		if (releaseGroupVersion === null) {
+			this.error(`Version isn't a valid semver string: '${releaseGroup.version}'`, {
+				exit: 1,
+			});
+		}
+
+		// Strips additional custom metadata from the source files before we call `changeset version`,
+		// because the changeset tools - like @changesets/cli - only work on canonical changesets.
+		const bumpType = await canonicalizeChangesets(releaseGroupRoot, this.logger);
+
+		// The `changeset version` command applies the changesets to the changelogs
+		ux.action.start("Running `changeset version`");
+		await execCommand("pnpm exec changeset version", { cwd: releaseGroupRoot });
+		ux.action.stop();
+
+		const packagesToCheck = releaseGroup.packages;
+
+		// restore the package versions that were changed by `changeset version`
+		await setVersion(packagesToCheck, releaseGroupVersion);
+
+		// Extract the version string from the SemVer object if provided
+		const versionString = versionOverride?.version;
+
+		// Calls processPackage on all packages.
+		ux.action.start("Processing changelog updates");
+		const processPromises: Promise<void>[] = [];
+		for (const pkg of packagesToCheck) {
+			processPromises.push(updateChangelogs(pkg, bumpType, versionString));
+		}
+		const results = await Promise.allSettled(processPromises);
+		const failures = results.filter((p) => p.status === "rejected");
+		if (failures.length > 0) {
+			this.error(
+				`Error processing packages; failure reasons:\n${failures
+					.map((p) => (p as PromiseRejectedResult).reason as string)
+					.join(", ")}`,
+				{ exit: 1 },
+			);
+		}
+		ux.action.stop();
+
+		this.log("Commit and open a PR!");
+	}
+}

build-tools/packages/build-cli/src/flags.ts

@@ -55,8 +55,8 @@ export const releaseGroupFlag = Flags.custom<ReleaseGroup>({
  * A re-usable CLI flag to parse release group names.
  */
 export const releaseGroupNameFlag = Flags.custom<ReleaseGroupName>({
-	required: true,
 	description: "The name of a release group.",
+	char: "g",
 	parse: async (input) => {
 		return input as ReleaseGroupName;
 	},

build-tools/packages/build-cli/src/library/changelogs.ts

@@ -0,0 +1,80 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import type { IPackage } from "@fluid-tools/build-infrastructure";
+import {
+	type VersionBumpType,
+	bumpVersionScheme,
+	isInternalVersionScheme,
+} from "@fluid-tools/version-tools";
+import { inc } from "semver";
+
+/**
+ * Escapes special regex characters in a string to make it safe for use in a RegExp.
+ */
+function escapeRegex(str: string): string {
+	return str.replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
+}
+
+/**
+ * Replaces all occurrences of a search string with a replacement string in a file.
+ * The search string is treated as a literal string, not a regex pattern.
+ *
+ * @param search - The literal string to search for (will be escaped for regex safety)
+ * @param replace - The string to replace matches with
+ * @param filePath - The path to the file to modify
+ * @throws Error if the file cannot be read or written
+ */
+async function replaceInFile(
+	search: string,
+	replace: string,
+	filePath: string,
+): Promise<void> {
+	try {
+		const content = await readFile(filePath, "utf8");
+		const escapedSearch = escapeRegex(search);
+		const newContent = content.replace(new RegExp(escapedSearch, "g"), replace);
+		await writeFile(filePath, newContent, "utf8");
+	} catch (error) {
+		throw new Error(
+			`Failed to replace "${search}" with "${replace}" in file ${filePath}: ${error}`,
+		);
+	}
+}
+
+export async function updateChangelogs(
+	pkg: IPackage,
+	bumpType: VersionBumpType,
+	version?: string,
+): Promise<void> {
+	if (pkg.isReleaseGroupRoot || pkg.isWorkspaceRoot) {
+		// No changelog for root packages.
+		return;
+	}
+	const { directory, version: pkgVersion } = pkg;
+
+	// This is the version that the changesets tooling calculates by default. It does a bump of the highest semver type
+	// in the changesets on the current version. We search for that version in the generated changelog and replace it
+	// with the one that we want.
+	const changesetsCalculatedVersion = isInternalVersionScheme(pkgVersion)
+		? bumpVersionScheme(pkgVersion, bumpType, "internal")
+		: inc(pkgVersion, bumpType);
+	const versionToUse = version ?? pkgVersion;
+
+	// Replace the changeset version with the correct version.
+	await replaceInFile(
+		`## ${changesetsCalculatedVersion}\n`,
+		`## ${versionToUse}\n`,
+		`${directory}/CHANGELOG.md`,
+	);
+
+	// For changelogs that had no changesets applied to them, add in a 'dependency updates only' section.
+	await replaceInFile(
+		`## ${versionToUse}\n\n## `,
+		`## ${versionToUse}\n\nDependency updates only.\n\n## `,
+		`${directory}/CHANGELOG.md`,
+	);
+}