diff --git a/docs/pages/usage/cli_generator.md b/docs/pages/usage/cli_generator.md index d036342f..8baf2762 100644 --- a/docs/pages/usage/cli_generator.md +++ b/docs/pages/usage/cli_generator.md @@ -67,7 +67,7 @@ The **Unregistered** commands will be used for generate code only, but will not ### Remove commands -When you unpick some commands in selection and regenerate the code, the unpicked commands will be removed. +When you unpick some commands in selection and regenerate the code by **Generate All**, the unpicked commands will be removed. **Generate Edit Only** never deletes generated code, it only rewrites the commands you edited. ![remove_commands](../../assets/recordings/cli_generator/remove_commands.gif) @@ -86,7 +86,14 @@ The sub folders of **aaz** in the module/extension represent each profiles. For ### Miss command models -When you open a module/extension, you may encounter `Miss command groups in aaz:...` error. That's because in your module/extension, some commands generated but in your local `aaz` repo the related command models don't exist. It can be resolved by two ways: +When you open a module/extension, some commands may be generated in your module/extension while the related command models don't exist in your local `aaz` repo, either the command model itself or the version generated in the module is missing. Commands whose model is missing are not displayed in the command tree, commands whose version is missing fall back to the latest version in your local `aaz` repo. A warning listing them is displayed in the generate dialog. + +Generation is not blocked: + +- **Generate Edit Only** keeps the code of those commands untouched. +- **Generate All** regenerates the whole `aaz` folder from your local `aaz` repo, so the code of those commands is **deleted** or **regenerated with another version**. + +It can be resolved by two ways: #### Sync the latest change of `aaz` repo in upstream diff --git a/src/aaz_dev/cli/controller/az_atomic_profile_builder.py b/src/aaz_dev/cli/controller/az_atomic_profile_builder.py index 31f60154..c43ab44b 100644 --- a/src/aaz_dev/cli/controller/az_atomic_profile_builder.py +++ b/src/aaz_dev/cli/controller/az_atomic_profile_builder.py @@ -128,6 +128,8 @@ def _build_command_from_aaz(self, *names, version_name, load_cfg=True): if v.name == version_name: version = v break + if not version and not load_cfg and aaz_cmd.versions: + version = aaz_cmd.versions[0] if not version: raise ResourceNotFind("Version '{}' of command '{}' not exist in AAZ".format(version_name, ' '.join(names))) diff --git a/src/aaz_dev/cli/controller/az_module_manager.py b/src/aaz_dev/cli/controller/az_module_manager.py index 944302d8..70f23eda 100644 --- a/src/aaz_dev/cli/controller/az_module_manager.py +++ b/src/aaz_dev/cli/controller/az_module_manager.py @@ -71,10 +71,11 @@ def load_module(self, mod_name): def update_module(self, mod_name, profiles, **kwargs): aaz_folder = self.get_aaz_path(mod_name) generators = {} - atomic_builder = AzAtomicProfileBuilder(mod_name=mod_name, by_patch=kwargs.pop('by_patch', False)) + by_patch = kwargs.pop('by_patch', False) + atomic_builder = AzAtomicProfileBuilder(mod_name=mod_name, by_patch=by_patch) for profile_name, profile in profiles.items(): profile = atomic_builder(profile) - generators[profile_name] = AzProfileGenerator(aaz_folder, profile) + generators[profile_name] = AzProfileGenerator(aaz_folder, profile, by_patch=by_patch) for generator in generators.values(): generator.generate() for generator in generators.values(): diff --git a/src/aaz_dev/cli/controller/az_profile_generator.py b/src/aaz_dev/cli/controller/az_profile_generator.py index ca32a69a..8bfe584b 100644 --- a/src/aaz_dev/cli/controller/az_profile_generator.py +++ b/src/aaz_dev/cli/controller/az_profile_generator.py @@ -11,10 +11,11 @@ class AzProfileGenerator: """Used to generate atomic layer command group""" - def __init__(self, aaz_folder, profile): + def __init__(self, aaz_folder, profile, by_patch=False): self.aaz_folder = aaz_folder self.profile = profile self.profile_folder_name = profile.profile_folder_name + self._by_patch = by_patch self._removed_folders = set() self._removed_files = set() self._modified_files = {} @@ -90,7 +91,8 @@ def _generate_by_command_group(self, profile_folder_name, command_group): for name in del_folders: self._delete_folder(profile_folder_name, *command_group_folder_names, name) - files = set() + # in patch mode files on disk are kept, so they must stay imported by __init__.py + files = set(cur_files) if self._by_patch else set() if command_group.commands: for command in command_group.commands.values(): assert command.names[:-1] == command_group.names, f"Invalid command name: {command.names}" @@ -171,12 +173,16 @@ def _get_path(self, *names): return os.path.join(self.aaz_folder, *names) def _delete_folder(self, *names): + if self._by_patch: + return path = self._get_path(*names) if os.path.exists(path): assert os.path.isdir(path), f'Invalid folder path {path}' self._removed_folders.add(path) def _delete_file(self, *names): + if self._by_patch: + return path = self._get_path(*names) if os.path.exists(path): assert os.path.isfile(path), f'Invalid file path {path}' diff --git a/src/aaz_dev/cli/tests/test_codegen_regressions.py b/src/aaz_dev/cli/tests/test_codegen_regressions.py index 4b6dd299..d664b1a7 100644 --- a/src/aaz_dev/cli/tests/test_codegen_regressions.py +++ b/src/aaz_dev/cli/tests/test_codegen_regressions.py @@ -6,13 +6,57 @@ from unittest.mock import patch from cli.api import _cmds +from cli.controller.az_atomic_profile_builder import AzAtomicProfileBuilder from cli.controller.az_module_manager import AzModuleManager +from cli.controller.az_profile_generator import AzProfileGenerator from swagger.model.specs import SwaggerSpecs, TypeSpecResourceProvider from utils.config import Config from utils.plane import PlaneEnum class CodegenRegressionTest(TestCase): + def test_patch_generation_preserves_files_missing_from_aaz(self): + with TemporaryDirectory() as folder: + group_folder = Path(folder) / "latest" / "test" + group_folder.mkdir(parents=True) + stale_file = group_folder / "_stale.py" + stale_file.write_text("stale\n", encoding="utf-8") + (group_folder / "_current.py").write_text("current\n", encoding="utf-8") + (group_folder / "__init__.py").write_text("", encoding="utf-8") + (group_folder / "__cmd_group.py").write_text("", encoding="utf-8") + profile = SimpleNamespace(profile_folder_name="latest", command_groups=None) + command_group = SimpleNamespace( + names=["test"], + command_groups=None, + commands={"current": SimpleNamespace(names=["test", "current"], cfg=None)}, + wait_command=None, + register_info=None, + help=SimpleNamespace(short="Test", long=None), + ) + + generator = AzProfileGenerator(folder, profile, by_patch=True) + generator._generate_by_command_group("latest", command_group) + generator.save() + + self.assertTrue(stale_file.exists()) + self.assertIn("from ._stale import *", (group_folder / "__init__.py").read_text(encoding="utf-8")) + + def test_patch_generation_accepts_removed_command_version(self): + with patch("cli.controller.az_atomic_profile_builder.AAZSpecsManager"): + builder = AzAtomicProfileBuilder("test", by_patch=True) + old_version = SimpleNamespace( + name="new", stage=None, examples=None, + resources=[SimpleNamespace(to_primitive=lambda: {"plane": PlaneEnum.Mgmt, "id": "/test"})], + ) + builder._aaz_spec_manager = SimpleNamespace(find_command=lambda *args: SimpleNamespace( + versions=[old_version], + help=SimpleNamespace(short="Test", lines=None), + )) + + command = builder._build_command_from_aaz("test", "show", version_name="old", load_cfg=False) + + self.assertEqual(command.version, "new") + def test_generate_rejects_incomplete_selections_before_updating_cli(self): resources = {"/test": {"v1": object()}} good = SimpleNamespace(name="Good", default_tag="v1", get_resource_map_by_tag=lambda _: resources) diff --git a/src/web/src/__tests__/unit/cli/CLIModGeneratorProfileCommandTree.test.ts b/src/web/src/__tests__/unit/cli/CLIModGeneratorProfileCommandTree.test.ts index 8166a7fe..12a53abf 100644 --- a/src/web/src/__tests__/unit/cli/CLIModGeneratorProfileCommandTree.test.ts +++ b/src/web/src/__tests__/unit/cli/CLIModGeneratorProfileCommandTree.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect } from "vitest"; import { ProfileCommandTree, + collectMissingVersionsInAaz, + decodeProfileCTCommand, initializeCommandTreeByModView, exportModViewProfile, } from "../../../views/cli/utils/commandTreeInitialization"; @@ -8,6 +10,51 @@ import { CLIModViewProfile } from "../../../views/cli/interfaces"; import { CLISpecsSimpleCommandTree } from "../../../views/cli/components/CLIModuleGenerator"; describe("CLIModGeneratorProfileCommandTree", () => { + it("should replace a removed selected version with the current aaz version", () => { + const command = decodeProfileCTCommand( + { + names: ["test", "show"], + help: { short: "Test" }, + versions: [{ name: "new", stage: "Stable", resources: [] }], + }, + true, + false, + true, + "old", + ); + + expect(command.selectedVersion).toBe("new"); + expect(command.missingVersionInAaz).toBe("old"); + expect( + collectMissingVersionsInAaz({ + name: "test-profile", + commandGroups: { + test: { id: "test", names: ["test"], commands: { show: command }, loading: false, selected: true }, + }, + }), + ).toEqual(["az test show (old -> new)"]); + }); + + it("should keep a selected version that still exists in aaz", () => { + const command = decodeProfileCTCommand( + { + names: ["test", "show"], + help: { short: "Test" }, + versions: [ + { name: "new", stage: "Stable", resources: [] }, + { name: "old", stage: "Stable", resources: [] }, + ], + }, + true, + false, + true, + "old", + ); + + expect(command.selectedVersion).toBe("old"); + expect(command.missingVersionInAaz).toBeUndefined(); + }); + describe("initializeCommandTreeByModView", () => { it("should initialize command tree with empty profile", () => { const profileName = "test-profile"; @@ -89,7 +136,7 @@ describe("CLIModGeneratorProfileCommandTree", () => { expect(result.commandGroups["test-group"].commands!["test-command"].registered).toBe(true); }); - it("should throw error for missing command groups in aaz", () => { + it("should ignore command groups missing from aaz", () => { const profileName = "test-profile"; const view: CLIModViewProfile = { name: "test-profile", @@ -108,9 +155,8 @@ describe("CLIModGeneratorProfileCommandTree", () => { }, }; - expect(() => { - initializeCommandTreeByModView(profileName, view, simpleTree); - }).toThrow("Miss command groups in aaz: `az missing-group`"); + expect(initializeCommandTreeByModView(profileName, view, simpleTree).commandGroups).toEqual({}); + expect(initializeCommandTreeByModView(profileName, view, simpleTree).missingInAaz).toEqual(["az missing-group"]); }); }); diff --git a/src/web/src/views/cli/components/CommandItem.tsx b/src/web/src/views/cli/components/CommandItem.tsx index 615230ef..9d181870 100644 --- a/src/web/src/views/cli/components/CommandItem.tsx +++ b/src/web/src/views/cli/components/CommandItem.tsx @@ -74,6 +74,7 @@ const CommandItem: React.FC = memo(({ command, onUpdateCommand return { ...oldCommand, selectedVersion: version, + missingVersionInAaz: undefined, modified: true, }; }); diff --git a/src/web/src/views/cli/components/GenerateDialog.tsx b/src/web/src/views/cli/components/GenerateDialog.tsx index 7c669933..4914a206 100644 --- a/src/web/src/views/cli/components/GenerateDialog.tsx +++ b/src/web/src/views/cli/components/GenerateDialog.tsx @@ -2,13 +2,19 @@ import { Alert, Button, Dialog, DialogActions, DialogContent, DialogTitle } from import { cliApi, errorHandlerApi } from "../../../services"; import { useAsyncOperation } from "../../../services/hooks"; import { AsyncOperationBanner } from "../../../components"; -import { exportModViewProfile, type ProfileCommandTree } from "../utils/commandTreeInitialization"; +import { + collectMissingVersionsInAaz, + exportModViewProfile, + type ProfileCommandTree, +} from "../utils/commandTreeInitialization"; import { type CLIModViewProfiles } from "../interfaces"; interface ProfileCommandTrees { [name: string]: ProfileCommandTree; } +const preview = (names: string[]) => names.slice(0, 3).join(", ") + (names.length > 3 ? ", ..." : ""); + interface GenerateDialogProps { repoName: string; moduleName: string; @@ -63,6 +69,9 @@ const GenerateDialog = (props: GenerateDialogProps) => { const isLoading = updateAllOperation.loading || updateModifiedOperation.loading; const error = updateAllOperation.error || updateModifiedOperation.error; + const trees = Object.values(props.profileCommandTrees); + const missingInAaz = trees.flatMap((tree) => tree.missingInAaz ?? []); + const missingVersionsInAaz = trees.flatMap(collectMissingVersionsInAaz); return ( @@ -70,6 +79,22 @@ const GenerateDialog = (props: GenerateDialogProps) => { + {!isLoading && (missingInAaz.length > 0 || missingVersionsInAaz.length > 0) && ( + + {[ + missingInAaz.length > 0 && + `${missingInAaz.length} generated command(s)/group(s) have no command model in the local aaz repo, ` + + `'Generate All' deletes their code: ${preview(missingInAaz)}`, + missingVersionsInAaz.length > 0 && + `${missingVersionsInAaz.length} generated command(s) have no model of their version in the local aaz ` + + `repo, 'Generate All' regenerates them with another version: ${preview(missingVersionsInAaz)}`, + "'Generate Edited Only' keeps them untouched.", + "See: https://azure.github.io/aaz-dev-tools/pages/usage/cli-generator/#miss-command-models.", + ] + .filter(Boolean) + .join("\n")} + + )} {error && ( {" "} diff --git a/src/web/src/views/cli/utils/commandTreeInitialization.ts b/src/web/src/views/cli/utils/commandTreeInitialization.ts index f481d795..b7e58d13 100644 --- a/src/web/src/views/cli/utils/commandTreeInitialization.ts +++ b/src/web/src/views/cli/utils/commandTreeInitialization.ts @@ -15,6 +15,7 @@ import { calculateSelected, type ProfileCTCommandGroup, type ProfileCTCommand, + type ProfileCTCommands, type ProfileCTCommandGroups, type ProfileCTCommandVersion, } from "./commandTreeUtils"; @@ -22,6 +23,8 @@ import { export interface ProfileCommandTree { name: string; commandGroups: ProfileCTCommandGroups; + // commands generated in the module but missing in local aaz repo + missingInAaz?: string[]; } export const decodeProfileCTCommandVersion = (response: any): ProfileCTCommandVersion => { @@ -50,15 +53,18 @@ export const decodeProfileCTCommand = ( }; if (selected) { let version: string | undefined; - if (selectedVersion !== undefined) { + let missingVersionInAaz: string | undefined; + if (selectedVersion !== undefined && versions?.some((value) => value.name === selectedVersion)) { version = selectedVersion; } else { version = versions ? versions[0].name : undefined; + missingVersionInAaz = selectedVersion; } return { ...command, selectedVersion: version, + missingVersionInAaz: missingVersionInAaz, }; } else { return command; @@ -80,9 +86,24 @@ const initializeCommandByModView = ( }; }; +const collectMissingInAaz = ( + view: { commands?: CLIModViewCommands; commandGroups?: CLIModViewCommandGroups } | undefined, + commands: ProfileCTCommands | undefined, + commandGroups: ProfileCTCommandGroups | undefined, + missingInAaz: string[], +) => { + Object.entries(view?.commands ?? {}) + .filter(([key]) => commands?.[key] === undefined) + .forEach(([, value]) => missingInAaz.push("az " + value.names.join(" "))); + Object.entries(view?.commandGroups ?? {}) + .filter(([key]) => commandGroups?.[key] === undefined) + .forEach(([, value]) => missingInAaz.push("az " + value.names.join(" "))); +}; + const initializeCommandGroupByModView = ( view: CLIModViewCommandGroup | undefined, simpleCommandGroup: CLISpecsSimpleCommandGroup, + missingInAaz: string[], ): ProfileCTCommandGroup => { const commands = simpleCommandGroup.commands !== undefined @@ -98,32 +119,11 @@ const initializeCommandGroupByModView = ( ? Object.fromEntries( Object.entries(simpleCommandGroup.commandGroups).map(([key, value]) => [ key, - initializeCommandGroupByModView(view?.commandGroups?.[key], value), + initializeCommandGroupByModView(view?.commandGroups?.[key], value, missingInAaz), ]), ) : undefined; - const leftCommands = Object.entries(view?.commands ?? {}) - .filter(([key, _]) => commands?.[key] === undefined) - .map(([_, value]) => value.names) - .map((names) => "`az " + names.join(" ") + "`"); - const leftCommandGroups = Object.entries(view?.commandGroups ?? {}) - .filter(([key, _]) => commandGroups?.[key] === undefined) - .map(([_, value]) => value.names) - .map((names) => "`az " + names.join(" ") + "`"); - const errors = []; - if (leftCommands.length > 0) { - errors.push(`Miss commands in aaz: ${leftCommands.join(", ")}`); - } - if (leftCommandGroups.length > 0) { - errors.push(`Miss command groups in aaz: ${leftCommandGroups.join(", ")}`); - } - if (errors.length > 0) { - throw new Error( - "\n" + - errors.join("\n") + - "\nSee: https://azure.github.io/aaz-dev-tools/pages/usage/cli-generator/#miss-command-models.", - ); - } + collectMissingInAaz(view, commands, commandGroups, missingInAaz); const selected = calculateSelected(commands ?? {}, commandGroups ?? {}); return { id: simpleCommandGroup.names.join("/"), @@ -141,27 +141,39 @@ export const initializeCommandTreeByModView = ( view: CLIModViewProfile | null, simpleTree: CLISpecsSimpleCommandTree, ): ProfileCommandTree => { + const missingInAaz: string[] = []; const commandGroups = Object.fromEntries( Object.entries(simpleTree.root.commandGroups).map(([key, value]) => [ key, - initializeCommandGroupByModView(view?.commandGroups?.[key], value), + initializeCommandGroupByModView(view?.commandGroups?.[key], value, missingInAaz), ]), ); - const leftCommandGroups = Object.entries(view?.commandGroups ?? {}) - .filter(([key, _]) => commandGroups?.[key] === undefined) - .map(([_, value]) => value.names) - .map((names) => "`az " + names.join(" ") + "`"); - if (leftCommandGroups.length > 0) { - throw new Error( - `\nMiss command groups in aaz: ${leftCommandGroups.join(", ")}\nSee: https://azure.github.io/aaz-dev-tools/pages/usage/cli-generator/#miss-command-models.`, - ); - } + collectMissingInAaz(view ?? undefined, undefined, commandGroups, missingInAaz); return { name: profileName, commandGroups: commandGroups, + missingInAaz: missingInAaz, }; }; +const collectMissingVersionsOfCommandGroup = (group: ProfileCTCommandGroup, missing: string[]) => { + Object.values(group.commands ?? {}).forEach((command) => { + if (command.selected && command.missingVersionInAaz !== undefined) { + missing.push(`az ${command.names.join(" ")} (${command.missingVersionInAaz} -> ${command.selectedVersion})`); + } + }); + Object.values(group.commandGroups ?? {}).forEach((subGroup) => + collectMissingVersionsOfCommandGroup(subGroup, missing), + ); +}; + +// versions generated in the module but missing in local aaz repo, they are replaced by the latest aaz version +export const collectMissingVersionsInAaz = (tree: ProfileCommandTree): string[] => { + const missing: string[] = []; + Object.values(tree.commandGroups).forEach((group) => collectMissingVersionsOfCommandGroup(group, missing)); + return missing; +}; + const exportModViewCommand = (command: ProfileCTCommand): CLIModViewCommand | undefined => { if (command.selectedVersion === undefined) { return undefined; diff --git a/src/web/src/views/cli/utils/commandTreeUtils.ts b/src/web/src/views/cli/utils/commandTreeUtils.ts index 7fc7e678..98505d69 100644 --- a/src/web/src/views/cli/utils/commandTreeUtils.ts +++ b/src/web/src/views/cli/utils/commandTreeUtils.ts @@ -21,6 +21,8 @@ interface ProfileCTCommand { names: string[]; versions?: ProfileCTCommandVersion[]; selectedVersion?: string; + // version generated in the module but missing in local aaz repo + missingVersionInAaz?: string; registered?: boolean; modified: boolean; loading: boolean;