From 7d0cbcec2aac1371f3be6d83abc942ff8119ae94 Mon Sep 17 00:00:00 2001 From: tadelesh <tadelesh.shi@live.cn> Date: Tue, 14 Jan 2025 16:45:34 +0800 Subject: [PATCH 1/7] bypass empty export error --- eng/tools/generator/cmd/v2/common/changelogProcessor.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/tools/generator/cmd/v2/common/changelogProcessor.go b/eng/tools/generator/cmd/v2/common/changelogProcessor.go index b3bd46e78834..a13a212192c1 100644 --- a/eng/tools/generator/cmd/v2/common/changelogProcessor.go +++ b/eng/tools/generator/cmd/v2/common/changelogProcessor.go @@ -168,7 +168,8 @@ func GetExportsFromTag(sdkRepo repo.SDKRepository, packagePath, tag string) (*ex // get exports result, err := exports.Get(packagePath) - if err != nil { + // bypass the error if the package doesn't contain any exports, return nil + if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") { return nil, err } From ecc6a7c1f681004d9e0087587eae9d575f254294 Mon Sep 17 00:00:00 2001 From: tadelesh <tadelesh.shi@live.cn> Date: Wed, 15 Jan 2025 23:47:32 +0800 Subject: [PATCH 2/7] support sub package --- .../cmd/v2/automation/automationCmd.go | 39 +++++--- .../cmd/v2/common/changelogProcessor.go | 6 +- .../generator/cmd/v2/common/fileProcessor.go | 18 +++- .../generator/cmd/v2/common/generation.go | 67 +++++++------ .../generator/cmd/v2/release/releaseCmd.go | 17 +++- .../generator/config/typespecRequests.go | 2 +- .../config/validate/localValidator.go | 2 +- .../template/typespec/CHANGELOG.md.tpl | 2 +- .../generator/template/typespec/README.md.tpl | 4 +- .../generator/template/typespec/ci.yml.tpl | 6 +- .../generator/template/typespec/go.mod.tpl | 2 +- eng/tools/generator/typespec/tspconfig.go | 23 +++-- .../generator/typespec/tspconfig_test.go | 93 ++++++++++++++++++- 13 files changed, 212 insertions(+), 69 deletions(-) diff --git a/eng/tools/generator/cmd/v2/automation/automationCmd.go b/eng/tools/generator/cmd/v2/automation/automationCmd.go index 1a4ae5c17dfe..a81cb8336c9f 100644 --- a/eng/tools/generator/cmd/v2/automation/automationCmd.go +++ b/eng/tools/generator/cmd/v2/automation/automationCmd.go @@ -125,24 +125,37 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline SpecRepoURL: input.RepoHTTPSURL, TypeSpecConfig: tsc, } - - module, err := tsc.GetModuleName() + + rpAndNamespaceName, err := tsc.GetRpAndPackageName() if err != nil { errorBuilder.add(err) continue } - packageModuleRelativePath := tsc.GetPackageModuleRelativePath() - if packageModuleRelativePath == "" { + + packageRelativePath := tsc.GetPackageRelativePath() + if packageRelativePath == "" { errorBuilder.add(fmt.Errorf("package module relative path not found in %s", tspconfigPath)) continue } + + moduleRelativePath := tsc.GetModuleRelativePath() + if moduleRelativePath == "" { + errorBuilder.add(fmt.Errorf("module relative path not found in %s", tspconfigPath)) + continue + } + + if !strings.HasPrefix(packageRelativePath, moduleRelativePath) { + errorBuilder.add(fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath)) + continue + } + namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{ - RPName: module[0], - NamespaceName: module[1], + RPName: rpAndNamespaceName[0], + NamespaceName: rpAndNamespaceName[1], SkipGenerateExample: true, GoVersion: ctx.goVersion, TspClientOptions: []string{"--debug"}, - }, packageModuleRelativePath) + }, packageRelativePath, moduleRelativePath) if err != nil { errorBuilder.add(err) continue @@ -151,8 +164,8 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline breaking := namespaceResult.Changelog.HasBreakingChanges() breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems() - srcFolder := filepath.Join(sdkRepo.Root(), packageModuleRelativePath) - apiViewArtifact := filepath.Join(sdkRepo.Root(), packageModuleRelativePath+".gosource") + srcFolder := filepath.Join(sdkRepo.Root(), packageRelativePath) + apiViewArtifact := filepath.Join(sdkRepo.Root(), packageRelativePath+".gosource") err := zipDirectory(srcFolder, apiViewArtifact) if err != nil { fmt.Println(err) @@ -160,16 +173,16 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline results = append(results, pipeline.PackageResult{ Version: namespaceResult.Version, - PackageName: packageModuleRelativePath, - Path: []string{packageModuleRelativePath}, - PackageFolder: packageModuleRelativePath, + PackageName: packageRelativePath, + Path: []string{packageRelativePath}, + PackageFolder: packageRelativePath, TypespecProject: []string{tspProjectFolder}, Changelog: &pipeline.Changelog{ Content: &content, HasBreakingChange: &breaking, BreakingChangeItems: &breakingChangeItems, }, - APIViewArtifact: packageModuleRelativePath + ".gosource", + APIViewArtifact: packageRelativePath + ".gosource", Language: "Go", }) diff --git a/eng/tools/generator/cmd/v2/common/changelogProcessor.go b/eng/tools/generator/cmd/v2/common/changelogProcessor.go index a13a212192c1..8060c6d4da54 100644 --- a/eng/tools/generator/cmd/v2/common/changelogProcessor.go +++ b/eng/tools/generator/cmd/v2/common/changelogProcessor.go @@ -30,8 +30,8 @@ const ( sdk_remote_url = "https://github.com/Azure/azure-sdk-for-go.git" ) -func GetAllVersionTags(packageModuleRelativePath string) ([]string, error) { - arr := strings.Split(packageModuleRelativePath, "/") +func GetAllVersionTags(moduleRelativePath string) ([]string, error) { + arr := strings.Split(moduleRelativePath, "/") log.Printf("Fetching all release tags from GitHub for RP: '%s' Package: '%s' ...", arr[len(arr)-2], arr[len(arr)-1]) client := http.Client{} res, err := client.Get(sdk_tag_fetch_url) @@ -52,7 +52,7 @@ func GetAllVersionTags(packageModuleRelativePath string) ([]string, error) { versionTag := make(map[string]string) for _, tag := range result { tagName := tag["ref"].(string) - if strings.Contains(tagName, packageModuleRelativePath+"/v") { + if strings.Contains(tagName, moduleRelativePath+"/v") { m := regexp.MustCompile(semver.SemVerRegex).FindString(tagName) versions = append(versions, m) versionTag[m] = tagName diff --git a/eng/tools/generator/cmd/v2/common/fileProcessor.go b/eng/tools/generator/cmd/v2/common/fileProcessor.go index 5cfb5f692da1..d8ed0f0e73f3 100644 --- a/eng/tools/generator/cmd/v2/common/fileProcessor.go +++ b/eng/tools/generator/cmd/v2/common/fileProcessor.go @@ -397,7 +397,6 @@ func AddChangelogToFile(changelog *Changelog, version *semver.Version, packageRo // replace `{{NewClientName}}` placeholder in README.md by first func name according to `^New.+Method$` pattern func ReplaceNewClientNamePlaceholder(packageRootPath string, exports exports.Content) error { - path := filepath.Join(packageRootPath, "README.md") var clientName string for _, k := range SortFuncItem(exports.Funcs) { v := exports.Funcs[k] @@ -407,6 +406,12 @@ func ReplaceNewClientNamePlaceholder(packageRootPath string, exports exports.Con } } + path := filepath.Join(packageRootPath, "README.md") + + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + b, err := os.ReadFile(path) if err != nil { return fmt.Errorf("cannot read from file '%s': %+v", path, err) @@ -420,9 +425,13 @@ func UpdateModuleDefinition(packageRootPath, packageModuleRelativePath string, v if version.Major() > 1 { path := filepath.Join(packageRootPath, "go.mod") + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + b, err := os.ReadFile(path) if err != nil { - return fmt.Errorf("cannot parse version from changelog") + return fmt.Errorf("cannot read go.mod") } lines := strings.Split(string(b), "\n") @@ -691,6 +700,11 @@ func ReplaceReadmeNewClientName(packageRootPath string, exports exports.Content) func ReplaceConstModuleVersion(packagePath string, newVersion string) error { path := filepath.Join(packagePath, "constants.go") + + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + data, err := os.ReadFile(path) if err != nil { return err diff --git a/eng/tools/generator/cmd/v2/common/generation.go b/eng/tools/generator/cmd/v2/common/generation.go index 3e229cb3d00e..8b37f6e513e5 100644 --- a/eng/tools/generator/cmd/v2/common/generation.go +++ b/eng/tools/generator/cmd/v2/common/generation.go @@ -235,7 +235,7 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate log.Printf("Start to generate changelog for package...") newExports, err := exports.Get(packagePath) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") { return nil, err } changelog, err := GetChangelogForPackage(oriExports, &newExports) @@ -374,9 +374,10 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate } } -func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, packageModuleRelativePath string) (*GenerateResult, error) { - packagePath := filepath.Join(ctx.SDKPath, packageModuleRelativePath) - changelogPath := filepath.Join(packagePath, ChangelogFileName) +func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, packageRelativePath string, moduleRelativePath string) (*GenerateResult, error) { + packagePath := filepath.Join(ctx.SDKPath, packageRelativePath) + modulePath := filepath.Join(ctx.SDKPath, moduleRelativePath) + changelogPath := filepath.Join(modulePath, ChangelogFileName) version, err := semver.NewVersion("0.1.0") if err != nil { @@ -393,21 +394,23 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa onBoard := false if _, err := os.Stat(changelogPath); os.IsNotExist(err) { onBoard = true - log.Printf("Package '%s' changelog not exist, do onboard process", packagePath) + log.Printf("Module '%s' changelog not exist, do onboard process", modulePath) if generateParam.SpecificPackageTitle == "" { generateParam.SpecificPackageTitle = strings.Title(generateParam.RPName) } log.Printf("Start to use template to generate new rp folder and basic package files...") sdkBasicInfo := map[string]any{ - "rpName": generateParam.RPName, - "packageName": generateParam.NamespaceName, - "packageTitle": generateParam.SpecificPackageTitle, - "packageVersion": version.String(), - "releaseDate": generateParam.ReleaseDate, - "goVersion": generateParam.GoVersion, - } - err = typespec.ParseTypeSpecTemplates(filepath.Join(ctx.SDKPath, "eng/tools/generator/template/typespec"), packagePath, sdkBasicInfo, nil) + "rpName": generateParam.RPName, + "packageName": generateParam.NamespaceName, + "moduleRelativePath": moduleRelativePath, + "serviceDir": strings.Replace(moduleRelativePath, "sdk/", "", 1), + "packageTitle": generateParam.SpecificPackageTitle, + "packageVersion": version.String(), + "releaseDate": generateParam.ReleaseDate, + "goVersion": generateParam.GoVersion, + } + err = typespec.ParseTypeSpecTemplates(filepath.Join(ctx.SDKPath, "eng/tools/generator/template/typespec"), modulePath, sdkBasicInfo, nil) if err != nil { return nil, err } @@ -448,13 +451,13 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa if !onBoard { log.Printf("Get ori exports for changelog generation...") - tags, err := GetAllVersionTags(packageModuleRelativePath) + tags, err := GetAllVersionTags(moduleRelativePath) if err != nil { return nil, err } if len(tags) == 0 { - return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/%s hasn't been released, it's supposed to OnBoard", packageModuleRelativePath) + return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/%s hasn't been released, it's supposed to OnBoard", packageRelativePath) } previousVersionTag := GetPreviousVersionTag(isCurrentPreview, tags) @@ -470,7 +473,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa log.Printf("Start to generate changelog for package...") newExports, err := exports.Get(packagePath) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") { return nil, err } changelog, err := GetChangelogForPackage(oriExports, &newExports) @@ -484,7 +487,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa var prl PullRequestLabel if onBoard { log.Printf("Replace {{NewClientName}} placeholder in the README.md ") - if err = ReplaceNewClientNamePlaceholder(packagePath, newExports); err != nil { + if err = ReplaceNewClientNamePlaceholder(modulePath, newExports); err != nil { return nil, err } @@ -500,7 +503,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } log.Printf("Replace version in CHANGELOG.md...") - if err = UpdateOnboardChangelogVersion(packagePath, version.String()); err != nil { + if err = UpdateOnboardChangelogVersion(modulePath, version.String()); err != nil { return nil, err } @@ -516,8 +519,18 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - log.Printf("##[command]Executing go mod tidy in %s\n", packagePath) - if err = ExecuteGo(packagePath, "mod", "tidy"); err != nil { + if packageRelativePath != moduleRelativePath { + // remove go.mod for sub package + goModPath := filepath.Join(packagePath, "go.mod") + if _, err := os.Stat(goModPath); !os.IsNotExist(err) { + if err = os.Remove(goModPath); err != nil { + return nil, err + } + } + } + + log.Printf("##[command]Executing go mod tidy in %s\n", modulePath) + if err = ExecuteGo(modulePath, "mod", "tidy"); err != nil { return nil, err } @@ -540,13 +553,13 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } log.Printf("Add changelog to file...") - changelogMd, err := AddChangelogToFile(changelog, version, packagePath, generateParam.ReleaseDate) + changelogMd, err := AddChangelogToFile(changelog, version, modulePath, generateParam.ReleaseDate) if err != nil { return nil, err } log.Printf("Update module definition if v2+...") - err = UpdateModuleDefinition(packagePath, packageModuleRelativePath, version) + err = UpdateModuleDefinition(packagePath, packageRelativePath, version) if err != nil { return nil, err } @@ -561,7 +574,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - baseModule := fmt.Sprintf("%s/%s", "github.com/Azure/azure-sdk-for-go", packageModuleRelativePath) + baseModule := fmt.Sprintf("%s/%s", "github.com/Azure/azure-sdk-for-go", packageRelativePath) if _, err := os.Stat(filepath.Join(packagePath, "fake")); !os.IsNotExist(err) && oldModuleVersion.Major() != version.Major() { log.Printf("Replace fake module v2+...") if err = ReplaceModule(version, packagePath, baseModule, ".go"); err != nil { @@ -578,12 +591,12 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } log.Printf("Replace README.md module...") - if err = replaceReadmeModule(packagePath, packageModuleRelativePath, version.String()); err != nil { + if err = replaceReadmeModule(modulePath, packageRelativePath, version.String()); err != nil { return nil, err } log.Printf("Replace README.md NewClient name...") - if err = ReplaceReadmeNewClientName(packagePath, newExports); err != nil { + if err = ReplaceReadmeNewClientName(modulePath, newExports); err != nil { return nil, err } @@ -614,8 +627,8 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - log.Printf("##[command]Executing go mod tidy in %s\n", packagePath) - if err = ExecuteGo(packagePath, "mod", "tidy"); err != nil { + log.Printf("##[command]Executing go mod tidy in %s\n", modulePath) + if err = ExecuteGo(modulePath, "mod", "tidy"); err != nil { return nil, err } diff --git a/eng/tools/generator/cmd/v2/release/releaseCmd.go b/eng/tools/generator/cmd/v2/release/releaseCmd.go index ceb0d3141420..d0b3ee3a1aa1 100644 --- a/eng/tools/generator/cmd/v2/release/releaseCmd.go +++ b/eng/tools/generator/cmd/v2/release/releaseCmd.go @@ -183,10 +183,21 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str if existTypeSpec { log.Printf("Generate SDK through TypeSpec...") - packageModuleRelativePath := generateCtx.TypeSpecConfig.GetPackageModuleRelativePath() - if packageModuleRelativePath == "" { + + packageRelativePath := generateCtx.TypeSpecConfig.GetPackageRelativePath() + if packageRelativePath == "" { return fmt.Errorf("package module relative path not found") } + + moduleRelativePath := generateCtx.TypeSpecConfig.GetModuleRelativePath() + if moduleRelativePath == "" { + return fmt.Errorf("module relative path not found in %s", generateCtx.TypeSpecConfig.Path) + } + + if !strings.HasPrefix(packageRelativePath, moduleRelativePath) { + return fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath) + } + result, err = generateCtx.GenerateForTypeSpec(&common.GenerateParam{ RPName: c.rpName, NamespaceName: c.namespaceName, @@ -198,7 +209,7 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str GoVersion: c.flags.GoVersion, TypeSpecEmitOption: c.flags.TypeSpecGoOption, TspClientOptions: c.flags.TspClientOption, - }, packageModuleRelativePath) + }, packageRelativePath, moduleRelativePath) } else { log.Printf("Generate SDK through AutoRest...") result, err = generateCtx.GenerateForSingleRPNamespace(&common.GenerateParam{ diff --git a/eng/tools/generator/config/typespecRequests.go b/eng/tools/generator/config/typespecRequests.go index e78671a30556..c268b8ad284f 100644 --- a/eng/tools/generator/config/typespecRequests.go +++ b/eng/tools/generator/config/typespecRequests.go @@ -48,7 +48,7 @@ func GetTypeSpecProjectsFromConfig(config *Config, specRoot string) (tspProjects if err != nil { return nil, err } - module, err := tspConfig.GetModuleName() + module, err := tspConfig.GetRpAndPackageName() if err != nil { return nil, err } diff --git a/eng/tools/generator/config/validate/localValidator.go b/eng/tools/generator/config/validate/localValidator.go index 9318e5cd34d4..75562855bc71 100644 --- a/eng/tools/generator/config/validate/localValidator.go +++ b/eng/tools/generator/config/validate/localValidator.go @@ -82,7 +82,7 @@ func getReadmeGoFromReadme(readme string) string { return strings.ReplaceAll(readme, readmeFilename, goReadmeFilename) } -func GetModuleName(content []byte) (string, string) { +func GetRpAndPackageName(content []byte) (string, string) { moduleExist := regexp.MustCompile(goReadmeModuleName).Match(content) if moduleExist { moduleName := regexp.MustCompile(goReadmeModuleName).FindString(string(content)) diff --git a/eng/tools/generator/template/typespec/CHANGELOG.md.tpl b/eng/tools/generator/template/typespec/CHANGELOG.md.tpl index 07ec06e679bc..cbe14a38128a 100644 --- a/eng/tools/generator/template/typespec/CHANGELOG.md.tpl +++ b/eng/tools/generator/template/typespec/CHANGELOG.md.tpl @@ -3,6 +3,6 @@ ## {{.packageVersion}} ({{.releaseDate}}) ### Other Changes -The package of `github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{{.rpName}}/{{.packageName}}` is using our [next generation design principles](https://azure.github.io/azure-sdk/general_introduction.html). +The package of `github.com/Azure/azure-sdk-for-go/sdk/{{.moduleRelativePath}}` is using our [next generation design principles](https://azure.github.io/azure-sdk/general_introduction.html). To learn more, please refer to our documentation [Quick Start](https://aka.ms/azsdk/go/mgmt). \ No newline at end of file diff --git a/eng/tools/generator/template/typespec/README.md.tpl b/eng/tools/generator/template/typespec/README.md.tpl index 1b8d1bb0f8b1..66bee1288771 100644 --- a/eng/tools/generator/template/typespec/README.md.tpl +++ b/eng/tools/generator/template/typespec/README.md.tpl @@ -2,7 +2,7 @@ The `{{.packageName}}` module provides operations for working with Azure {{.packageTitle}}. -[Source code](https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/resourcemanager/{{.rpName}}/{{.packageName}}) +[Source code](https://github.com/Azure/azure-sdk-for-go/tree/main/{{.moduleRelativePath}}) # Getting started @@ -18,7 +18,7 @@ This project uses [Go modules](https://github.com/golang/go/wiki/Modules) for ve Install the Azure {{.packageTitle}} module: ```sh -go get github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{{.rpName}}/{{.packageName}} +go get github.com/Azure/azure-sdk-for-go/{{.moduleRelativePath}} ``` ## Authorization diff --git a/eng/tools/generator/template/typespec/ci.yml.tpl b/eng/tools/generator/template/typespec/ci.yml.tpl index b699dc9a3e14..f5ec43fee3d0 100644 --- a/eng/tools/generator/template/typespec/ci.yml.tpl +++ b/eng/tools/generator/template/typespec/ci.yml.tpl @@ -8,7 +8,7 @@ trigger: - release/* paths: include: - - sdk/resourcemanager/{{.rpName}}/{{.packageName}}/ + - {{.moduleRelativePath}}/ pr: branches: @@ -19,10 +19,10 @@ pr: - release/* paths: include: - - sdk/resourcemanager/{{.rpName}}/{{.packageName}}/ + - {{.moduleRelativePath}}/ extends: template: /eng/pipelines/templates/jobs/archetype-sdk-client.yml parameters: IncludeRelease: true - ServiceDirectory: 'resourcemanager/{{.rpName}}/{{.packageName}}' + ServiceDirectory: '{{.serviceDir}}' diff --git a/eng/tools/generator/template/typespec/go.mod.tpl b/eng/tools/generator/template/typespec/go.mod.tpl index 39e1daf4f726..117db716062b 100644 --- a/eng/tools/generator/template/typespec/go.mod.tpl +++ b/eng/tools/generator/template/typespec/go.mod.tpl @@ -1,4 +1,4 @@ -module github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{{.rpName}}/{{.packageName}} +module github.com/Azure/azure-sdk-for-go/{{.moduleRelativePath}} go {{.goVersion}} diff --git a/eng/tools/generator/typespec/tspconfig.go b/eng/tools/generator/typespec/tspconfig.go index bf402eb24678..5b95902b7bfa 100644 --- a/eng/tools/generator/typespec/tspconfig.go +++ b/eng/tools/generator/typespec/tspconfig.go @@ -117,7 +117,19 @@ func ParseTypeSpecConfig(tspconfigPath string) (*TypeSpecConfig, error) { return &tspConfig, err } -func (tc *TypeSpecConfig) GetPackageModuleRelativePath() string { +func (tc *TypeSpecConfig) GetPackageRelativePath() string { + goConfig := tc.Options["@azure-tools/typespec-go"].(map[string]interface{}) + if goConfig["package-dir"] == nil { + return tc.GetModuleRelativePath() + } else { + if goConfig["service-dir"] == nil { + goConfig["service-dir"] = tc.Parameters["service-dir"].(map[string]interface {})["default"] + } + return goConfig["service-dir"].(string) + "/" + goConfig["package-dir"].(string) + } +} + +func (tc *TypeSpecConfig) GetModuleRelativePath() string { goConfig := tc.Options["@azure-tools/typespec-go"].(map[string]interface{}) if goConfig["module"] == nil { return "" @@ -180,9 +192,9 @@ func (tc TypeSpecConfig) ExistEmitOption(emit string) bool { return err == nil } -// GetModuleName return [rpName, packageName] -// module: github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{rpName}/{packageName} -func (tc TypeSpecConfig) GetModuleName() ([2]string, error) { +// GetRpAndPackageName return [rpName, packageName] +// module: github.com/Azure/azure-sdk-for-go/sdk/.../{rpName}/{packageName} +func (tc TypeSpecConfig) GetRpAndPackageName() ([2]string, error) { option, err := tc.EmitOption(string(TypeSpec_GO)) if err != nil { return [2]string{}, err @@ -191,9 +203,6 @@ func (tc TypeSpecConfig) GetModuleName() ([2]string, error) { module := (option.(map[string]any))["module"].(string) s := strings.Split(module, "/") l := len(s) - if l != 7 { - return [2]string{}, fmt.Errorf("module is invalid and must be in the format of `github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{rpName}/{packageName}`") - } if !strings.Contains(s[l-1], "arm") && !strings.Contains(s[l-1], "az") { return [2]string{}, fmt.Errorf("packageName is invalid and must start with `arm` or `az`") } diff --git a/eng/tools/generator/typespec/tspconfig_test.go b/eng/tools/generator/typespec/tspconfig_test.go index 5800b3355f44..332b322c5058 100644 --- a/eng/tools/generator/typespec/tspconfig_test.go +++ b/eng/tools/generator/typespec/tspconfig_test.go @@ -7,14 +7,14 @@ import ( "github.com/stretchr/testify/assert" ) -func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { +func TestTypeSpecConfig_GetPackageRelativePath(t *testing.T) { tests := []struct { name string config typespec.TypeSpecConfig expected string }{ { - name: "Valid module path", + name: "Package path from module", config: typespec.TypeSpecConfig{ TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ Options: map[string]any{ @@ -27,7 +27,75 @@ func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { expected: "sdk/messaging/eventgrid/azsystemevents", }, { - name: "Module path with placeholders", + name: "Package path from module with placeholder", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/{service-dir}/armcompute", + "service-dir": "sdk/resourcemanager/compute", + }, + }, + }, + }, + expected: "sdk/resourcemanager/compute/armcompute", + }, + { + name: "Empty package path", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{}, + }, + }, + }, + expected: "", + }, + { + name: "Package path from service and package dir", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/{service-dir}/azadmin", + "service-dir": "sdk/security/keyvault", + "package-dir": "azadmin/backup", + }, + }, + }, + }, + expected: "sdk/security/keyvault/azadmin/backup", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.config.GetPackageRelativePath()) + }) + } +} + +func TestTypeSpecConfig_GetModuleRelativePath(t *testing.T) { + tests := []struct { + name string + config typespec.TypeSpecConfig + expected string + }{ + { + name: "Normal", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/sdk/messaging/eventgrid/azsystemevents", + }, + }, + }, + }, + expected: "sdk/messaging/eventgrid/azsystemevents", + }, + { + name: "Module with placeholder", config: typespec.TypeSpecConfig{ TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ Options: map[string]any{ @@ -42,7 +110,7 @@ func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { expected: "sdk/resourcemanager/compute/armcompute", }, { - name: "Module path without module key", + name: "Empty module", config: typespec.TypeSpecConfig{ TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ Options: map[string]any{ @@ -52,11 +120,26 @@ func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { }, expected: "", }, + { + name: "Module different from package path", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/{service-dir}/azadmin", + "service-dir": "sdk/security/keyvault", + "package-dir": "azadmin/backup", + }, + }, + }, + }, + expected: "sdk/security/keyvault/azadmin", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, tt.config.GetPackageModuleRelativePath()) + assert.Equal(t, tt.expected, tt.config.GetModuleRelativePath()) }) } } From 74f536313f398a7f9fa97627d44c6ded3817d6b3 Mon Sep 17 00:00:00 2001 From: JiaqiZhang-Dev <jiaqzhang@microsoft.com> Date: Thu, 6 Feb 2025 11:44:48 +0800 Subject: [PATCH 3/7] Compatible with the case where module is empty --- .../cmd/v2/automation/automationCmd.go | 31 +++++++++++++----- .../generator/cmd/v2/common/constants.go | 2 ++ .../generator/cmd/v2/common/fileProcessor.go | 32 +++++++++++++++++++ .../cmd/v2/common/fileProcessor_test.go | 18 +++++++++++ .../generator/cmd/v2/common/generation.go | 8 +++-- eng/tools/generator/typespec/tspconfig.go | 23 ++++++++++--- eng/tools/generator/typespec/typespec-go.go | 4 ++- 7 files changed, 102 insertions(+), 16 deletions(-) diff --git a/eng/tools/generator/cmd/v2/automation/automationCmd.go b/eng/tools/generator/cmd/v2/automation/automationCmd.go index a81cb8336c9f..cdf7316daffc 100644 --- a/eng/tools/generator/cmd/v2/automation/automationCmd.go +++ b/eng/tools/generator/cmd/v2/automation/automationCmd.go @@ -125,12 +125,6 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline SpecRepoURL: input.RepoHTTPSURL, TypeSpecConfig: tsc, } - - rpAndNamespaceName, err := tsc.GetRpAndPackageName() - if err != nil { - errorBuilder.add(err) - continue - } packageRelativePath := tsc.GetPackageRelativePath() if packageRelativePath == "" { @@ -138,10 +132,20 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline continue } + isModule := true moduleRelativePath := tsc.GetModuleRelativePath() + // if module relative path is not provided, find it from the sdk path by go.mod if moduleRelativePath == "" { - errorBuilder.add(fmt.Errorf("module relative path not found in %s", tspconfigPath)) - continue + isModule = false + val, err := common.FindModuleDirByGoMod(filepath.Join(generateCtx.SDKPath, packageRelativePath)) + if err != nil { + return nil, err + } + moduleRelativePath, err = filepath.Rel(generateCtx.SDKPath, val) + if err != nil { + return nil, err + } + moduleRelativePath = filepath.ToSlash(moduleRelativePath) } if !strings.HasPrefix(packageRelativePath, moduleRelativePath) { @@ -149,12 +153,23 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline continue } + if packageRelativePath != moduleRelativePath { + isModule = false + } + + rpAndNamespaceName, err := tsc.GetRpAndPackageNameByModule(moduleRelativePath) + if err != nil { + errorBuilder.add(err) + continue + } + namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{ RPName: rpAndNamespaceName[0], NamespaceName: rpAndNamespaceName[1], SkipGenerateExample: true, GoVersion: ctx.goVersion, TspClientOptions: []string{"--debug"}, + IsModule: isModule, }, packageRelativePath, moduleRelativePath) if err != nil { errorBuilder.add(err) diff --git a/eng/tools/generator/cmd/v2/common/constants.go b/eng/tools/generator/cmd/v2/common/constants.go index 836c12d3662b..d6dbcf314b93 100644 --- a/eng/tools/generator/cmd/v2/common/constants.go +++ b/eng/tools/generator/cmd/v2/common/constants.go @@ -5,4 +5,6 @@ package common const ( ChangelogFileName = "CHANGELOG.md" + GoModFileName = "go.mod" + SdkRootPath = "sdk" ) diff --git a/eng/tools/generator/cmd/v2/common/fileProcessor.go b/eng/tools/generator/cmd/v2/common/fileProcessor.go index d8ed0f0e73f3..c9c703e7c2c6 100644 --- a/eng/tools/generator/cmd/v2/common/fileProcessor.go +++ b/eng/tools/generator/cmd/v2/common/fileProcessor.go @@ -138,6 +138,9 @@ func ReadV2ModuleNameToGetNamespace(path string) (map[string][]PackageInfo, erro // remove all sdk generated files in given path func CleanSDKGeneratedFiles(path string) error { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } log.Printf("Removing all sdk generated files in '%s'...", path) return filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { if err != nil { @@ -803,3 +806,32 @@ func importPath(s *ast.ImportSpec) string { } return t } + +// Walks the sdk directory to find module based on a go.mod file +func FindModuleDirByGoMod(root string) (string, error) { + path := root + curLevel := 0 + maxLevel := 5 + for !strings.HasSuffix(path, SdkRootPath) && curLevel < maxLevel { + if _, err := os.Stat(path); os.IsNotExist(err) { + path = filepath.Dir(path) + curLevel++ + continue + } + entries, err := os.ReadDir(path) + if err != nil { + return "", err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + if entry.Name() == GoModFileName { + return path, nil + } + } + path = filepath.Dir(path) + curLevel++ + } + return "", fmt.Errorf("not found module, package path:%s", root) +} diff --git a/eng/tools/generator/cmd/v2/common/fileProcessor_test.go b/eng/tools/generator/cmd/v2/common/fileProcessor_test.go index 63219d2274f3..cdb981a48575 100644 --- a/eng/tools/generator/cmd/v2/common/fileProcessor_test.go +++ b/eng/tools/generator/cmd/v2/common/fileProcessor_test.go @@ -4,11 +4,16 @@ package common import ( + "fmt" + "os" + "path/filepath" "testing" + "github.com/Azure/azure-sdk-for-go/eng/tools/generator/repo" "github.com/Azure/azure-sdk-for-go/eng/tools/internal/delta" "github.com/Azure/azure-sdk-for-go/eng/tools/internal/exports" "github.com/Azure/azure-sdk-for-go/eng/tools/internal/report" + "github.com/Azure/azure-sdk-for-go/eng/tools/internal/utils" "github.com/stretchr/testify/assert" ) @@ -122,3 +127,16 @@ func TestCalculateNewVersion(t *testing.T) { assert.Equal(t, newVersion.String(), "1.2.0-beta.2") assert.Equal(t, BetaLabel, prl) } + +func TestFindModule(t *testing.T) { + cwd, err := os.Getwd() + assert.NoError(t, err) + sdkRoot := utils.NormalizePath(cwd) + sdkRepo, err := repo.OpenSDKRepository(sdkRoot) + assert.NoError(t, err) + module, err := FindModuleDirByGoMod(fmt.Sprintf("%s/%s", filepath.ToSlash(sdkRepo.Root()), "sdk/security/keyvault/azadmin/settings")) + assert.NoError(t, err) + moduleRelativePath, err := filepath.Rel(sdkRepo.Root(), module) + assert.NoError(t, err) + assert.Equal(t, "sdk/security/keyvault/azadmin", filepath.ToSlash(moduleRelativePath)) +} diff --git a/eng/tools/generator/cmd/v2/common/generation.go b/eng/tools/generator/cmd/v2/common/generation.go index 8b37f6e513e5..f9600ec16cde 100644 --- a/eng/tools/generator/cmd/v2/common/generation.go +++ b/eng/tools/generator/cmd/v2/common/generation.go @@ -55,6 +55,7 @@ type GenerateParam struct { ForceStableVersion bool TypeSpecEmitOption string TspClientOptions []string + IsModule bool } func (ctx *GenerateContext) GenerateForAutomation(readme, repo, goVersion string) ([]GenerateResult, []error) { @@ -424,7 +425,10 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa log.Printf("Start to run `tsp-client init` to generate the code...") defaultModuleVersion := version.String() - emitOption := fmt.Sprintf("module-version=%s", defaultModuleVersion) + emitOption := "" + if generateParam.IsModule { + emitOption = fmt.Sprintf("module-version=%s", defaultModuleVersion) + } if generateParam.TypeSpecEmitOption != "" { emitOption = fmt.Sprintf("%s;%s", emitOption, generateParam.TypeSpecEmitOption) } @@ -519,7 +523,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - if packageRelativePath != moduleRelativePath { + if !generateParam.IsModule { // remove go.mod for sub package goModPath := filepath.Join(packagePath, "go.mod") if _, err := os.Stat(goModPath); !os.IsNotExist(err) { diff --git a/eng/tools/generator/typespec/tspconfig.go b/eng/tools/generator/typespec/tspconfig.go index 5b95902b7bfa..671181e970d3 100644 --- a/eng/tools/generator/typespec/tspconfig.go +++ b/eng/tools/generator/typespec/tspconfig.go @@ -89,9 +89,10 @@ func ParseTypeSpecConfig(tspconfigPath string) (*TypeSpecConfig, error) { } goOption := emitOption.(map[string]any) - module, ok := goOption["module"].(string) - if !ok { - return nil, fmt.Errorf("the module must be set in %s option", TypeSpec_GO) + module, moduleOK := goOption["module"].(string) + _, packageOK := goOption["package-dir"].(string) + if !moduleOK && !packageOK { + return nil, fmt.Errorf("the module or package must be set in %s option", TypeSpec_GO) } if strings.Contains(module, "{service-dir}") { @@ -123,7 +124,7 @@ func (tc *TypeSpecConfig) GetPackageRelativePath() string { return tc.GetModuleRelativePath() } else { if goConfig["service-dir"] == nil { - goConfig["service-dir"] = tc.Parameters["service-dir"].(map[string]interface {})["default"] + goConfig["service-dir"] = tc.Parameters["service-dir"].(map[string]interface{})["default"] } return goConfig["service-dir"].(string) + "/" + goConfig["package-dir"].(string) } @@ -199,10 +200,22 @@ func (tc TypeSpecConfig) GetRpAndPackageName() ([2]string, error) { if err != nil { return [2]string{}, err } + goOption := option.(map[string]any) + module, ok := goOption["module"].(string) + if !ok || len(module) == 0 { + return [2]string{}, nil + } + return tc.GetRpAndPackageNameByModule(module) +} - module := (option.(map[string]any))["module"].(string) +// GetRpAndPackageName return [rpName, packageName] +// module: github.com/Azure/azure-sdk-for-go/sdk/.../{rpName}/{packageName} +func (tc TypeSpecConfig) GetRpAndPackageNameByModule(module string) ([2]string, error) { s := strings.Split(module, "/") l := len(s) + if l < 2 { + return [2]string{}, fmt.Errorf("module is invalid") + } if !strings.Contains(s[l-1], "arm") && !strings.Contains(s[l-1], "az") { return [2]string{}, fmt.Errorf("packageName is invalid and must start with `arm` or `az`") } diff --git a/eng/tools/generator/typespec/typespec-go.go b/eng/tools/generator/typespec/typespec-go.go index a311728d0a8e..8c0fe99a7e28 100644 --- a/eng/tools/generator/typespec/typespec-go.go +++ b/eng/tools/generator/typespec/typespec-go.go @@ -2,6 +2,7 @@ package typespec import ( "errors" + "log" "regexp" "github.com/goccy/go-yaml" @@ -56,7 +57,8 @@ var ( func (o *GoEmitterOptions) Validate() error { if o.Module == "" { - return ErrModuleEmpty + log.Printf("typesepec-go option `module` is empty") + return nil } matched := regexp.MustCompile(moduleRegex).MatchString(o.Module) From d81b51ffd50c7d5bf128ca5886ad25316ce8bd84 Mon Sep 17 00:00:00 2001 From: JiaqiZhang-Dev <jiaqzhang@microsoft.com> Date: Thu, 6 Feb 2025 14:44:51 +0800 Subject: [PATCH 4/7] refine generate function --- .../cmd/v2/automation/automationCmd.go | 138 +++++++----------- .../generator/cmd/v2/common/generation.go | 91 +++++++++--- .../generator/cmd/v2/release/releaseCmd.go | 17 +-- 3 files changed, 117 insertions(+), 129 deletions(-) diff --git a/eng/tools/generator/cmd/v2/automation/automationCmd.go b/eng/tools/generator/cmd/v2/automation/automationCmd.go index cdf7316daffc..14578e6a051f 100644 --- a/eng/tools/generator/cmd/v2/automation/automationCmd.go +++ b/eng/tools/generator/cmd/v2/automation/automationCmd.go @@ -115,97 +115,57 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline continue } - if ok := tsc.ExistEmitOption(string(typespec.TypeSpec_GO)); ok { - log.Printf("Start to process typespec project: %s", tspProjectFolder) - generateCtx := common.GenerateContext{ - SDKPath: sdkRepo.Root(), - SDKRepo: &sdkRepo, - SpecPath: ctx.specRoot, - SpecCommitHash: ctx.commitHash, - SpecRepoURL: input.RepoHTTPSURL, - TypeSpecConfig: tsc, - } - - packageRelativePath := tsc.GetPackageRelativePath() - if packageRelativePath == "" { - errorBuilder.add(fmt.Errorf("package module relative path not found in %s", tspconfigPath)) - continue - } - - isModule := true - moduleRelativePath := tsc.GetModuleRelativePath() - // if module relative path is not provided, find it from the sdk path by go.mod - if moduleRelativePath == "" { - isModule = false - val, err := common.FindModuleDirByGoMod(filepath.Join(generateCtx.SDKPath, packageRelativePath)) - if err != nil { - return nil, err - } - moduleRelativePath, err = filepath.Rel(generateCtx.SDKPath, val) - if err != nil { - return nil, err - } - moduleRelativePath = filepath.ToSlash(moduleRelativePath) - } - - if !strings.HasPrefix(packageRelativePath, moduleRelativePath) { - errorBuilder.add(fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath)) - continue - } - - if packageRelativePath != moduleRelativePath { - isModule = false - } - - rpAndNamespaceName, err := tsc.GetRpAndPackageNameByModule(moduleRelativePath) - if err != nil { - errorBuilder.add(err) - continue - } - - namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{ - RPName: rpAndNamespaceName[0], - NamespaceName: rpAndNamespaceName[1], - SkipGenerateExample: true, - GoVersion: ctx.goVersion, - TspClientOptions: []string{"--debug"}, - IsModule: isModule, - }, packageRelativePath, moduleRelativePath) - if err != nil { - errorBuilder.add(err) - continue - } else { - content := namespaceResult.ChangelogMD - breaking := namespaceResult.Changelog.HasBreakingChanges() - breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems() - - srcFolder := filepath.Join(sdkRepo.Root(), packageRelativePath) - apiViewArtifact := filepath.Join(sdkRepo.Root(), packageRelativePath+".gosource") - err := zipDirectory(srcFolder, apiViewArtifact) - if err != nil { - fmt.Println(err) - } - - results = append(results, pipeline.PackageResult{ - Version: namespaceResult.Version, - PackageName: packageRelativePath, - Path: []string{packageRelativePath}, - PackageFolder: packageRelativePath, - TypespecProject: []string{tspProjectFolder}, - Changelog: &pipeline.Changelog{ - Content: &content, - HasBreakingChange: &breaking, - BreakingChangeItems: &breakingChangeItems, - }, - APIViewArtifact: packageRelativePath + ".gosource", - Language: "Go", - }) - - log.Printf("Finish processing typespec file: %s", tspconfigPath) - } - } else { + if ok := tsc.ExistEmitOption(string(typespec.TypeSpec_GO)); !ok { errorBuilder.add(fmt.Errorf("`@azure-tools/typespec-go` option not found in %s, it is required, please refer to `https://aka.ms/azsdk/tspconfig-sample-mpg` to configure it", tspconfigPath)) + continue + } + log.Printf("Start to process typespec project: %s", tspProjectFolder) + generateCtx := common.GenerateContext{ + SDKPath: sdkRepo.Root(), + SDKRepo: &sdkRepo, + SpecPath: ctx.specRoot, + SpecCommitHash: ctx.commitHash, + SpecRepoURL: input.RepoHTTPSURL, + TypeSpecConfig: tsc, + } + + namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{ + SkipGenerateExample: true, + GoVersion: ctx.goVersion, + TspClientOptions: []string{"--debug"}, + }) + if err != nil { + errorBuilder.add(err) + continue + } + content := namespaceResult.ChangelogMD + breaking := namespaceResult.Changelog.HasBreakingChanges() + breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems() + packageRelativePath := namespaceResult.PackageRelativePath + + srcFolder := filepath.Join(sdkRepo.Root(), packageRelativePath) + apiViewArtifact := filepath.Join(sdkRepo.Root(), packageRelativePath+".gosource") + err = zipDirectory(srcFolder, apiViewArtifact) + if err != nil { + fmt.Println(err) } + + results = append(results, pipeline.PackageResult{ + Version: namespaceResult.Version, + PackageName: packageRelativePath, + Path: []string{packageRelativePath}, + PackageFolder: packageRelativePath, + TypespecProject: []string{tspProjectFolder}, + Changelog: &pipeline.Changelog{ + Content: &content, + HasBreakingChange: &breaking, + BreakingChangeItems: &breakingChangeItems, + }, + APIViewArtifact: packageRelativePath + ".gosource", + Language: "Go", + }) + + log.Printf("Finish processing typespec file: %s", tspconfigPath) } // autorest diff --git a/eng/tools/generator/cmd/v2/common/generation.go b/eng/tools/generator/cmd/v2/common/generation.go index f9600ec16cde..442426cc4c89 100644 --- a/eng/tools/generator/cmd/v2/common/generation.go +++ b/eng/tools/generator/cmd/v2/common/generation.go @@ -32,13 +32,14 @@ type GenerateContext struct { } type GenerateResult struct { - Version string - RPName string - PackageName string - PackageAbsPath string - Changelog Changelog - ChangelogMD string - PullRequestLabels string + Version string + RPName string + PackageName string + PackageAbsPath string + Changelog Changelog + ChangelogMD string + PullRequestLabels string + PackageRelativePath string } type GenerateParam struct { @@ -375,7 +376,47 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate } } -func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, packageRelativePath string, moduleRelativePath string) (*GenerateResult, error) { +func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (*GenerateResult, error) { + isModule := true + + packageRelativePath := ctx.TypeSpecConfig.GetPackageRelativePath() + if packageRelativePath == "" { + return nil, fmt.Errorf("package module relative path not found in %s", ctx.TypeSpecConfig.Path) + } + + moduleRelativePath := ctx.TypeSpecConfig.GetModuleRelativePath() + // if module relative path is not provided, find it from the sdk path by go.mod + if moduleRelativePath == "" { + isModule = false + val, err := FindModuleDirByGoMod(filepath.Join(ctx.SDKPath, packageRelativePath)) + if err != nil { + return nil, err + } + moduleRelativePath, err = filepath.Rel(ctx.SDKPath, val) + if err != nil { + return nil, err + } + moduleRelativePath = filepath.ToSlash(moduleRelativePath) + } + + if !strings.HasPrefix(packageRelativePath, moduleRelativePath) { + return nil, fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath) + } + + if packageRelativePath != moduleRelativePath { + isModule = false + } + + // if rp name and namespace name are not provided, extract them from the module path + if len(generateParam.RPName) == 0 && len(generateParam.NamespaceName) == 0 { + rpAndNamespaceName, err := ctx.TypeSpecConfig.GetRpAndPackageNameByModule(moduleRelativePath) + if err != nil { + return nil, err + } + generateParam.RPName = rpAndNamespaceName[0] + generateParam.NamespaceName = rpAndNamespaceName[1] + } + packagePath := filepath.Join(ctx.SDKPath, packageRelativePath) modulePath := filepath.Join(ctx.SDKPath, moduleRelativePath) changelogPath := filepath.Join(modulePath, ChangelogFileName) @@ -426,7 +467,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa log.Printf("Start to run `tsp-client init` to generate the code...") defaultModuleVersion := version.String() emitOption := "" - if generateParam.IsModule { + if isModule { emitOption = fmt.Sprintf("module-version=%s", defaultModuleVersion) } if generateParam.TypeSpecEmitOption != "" { @@ -523,7 +564,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - if !generateParam.IsModule { + if isModule { // remove go.mod for sub package goModPath := filepath.Join(packagePath, "go.mod") if _, err := os.Stat(goModPath); !os.IsNotExist(err) { @@ -539,13 +580,14 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } return &GenerateResult{ - Version: version.String(), - RPName: generateParam.RPName, - PackageName: generateParam.NamespaceName, - PackageAbsPath: packagePath, - Changelog: *changelog, - ChangelogMD: changelog.ToCompactMarkdown() + "\n" + changelog.GetChangeSummary(), - PullRequestLabels: string(prl), + Version: version.String(), + RPName: generateParam.RPName, + PackageName: generateParam.NamespaceName, + PackageAbsPath: packagePath, + Changelog: *changelog, + ChangelogMD: changelog.ToCompactMarkdown() + "\n" + changelog.GetChangeSummary(), + PullRequestLabels: string(prl), + PackageRelativePath: packageRelativePath, }, nil } else { log.Printf("Calculate new version...") @@ -637,13 +679,14 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } return &GenerateResult{ - Version: version.String(), - RPName: generateParam.RPName, - PackageName: generateParam.NamespaceName, - PackageAbsPath: packagePath, - Changelog: *changelog, - ChangelogMD: changelogMd + "\n" + changelog.GetChangeSummary(), - PullRequestLabels: string(prl), + Version: version.String(), + RPName: generateParam.RPName, + PackageName: generateParam.NamespaceName, + PackageAbsPath: packagePath, + Changelog: *changelog, + ChangelogMD: changelogMd + "\n" + changelog.GetChangeSummary(), + PullRequestLabels: string(prl), + PackageRelativePath: packageRelativePath, }, nil } } diff --git a/eng/tools/generator/cmd/v2/release/releaseCmd.go b/eng/tools/generator/cmd/v2/release/releaseCmd.go index d0b3ee3a1aa1..42e0a866c223 100644 --- a/eng/tools/generator/cmd/v2/release/releaseCmd.go +++ b/eng/tools/generator/cmd/v2/release/releaseCmd.go @@ -183,21 +183,6 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str if existTypeSpec { log.Printf("Generate SDK through TypeSpec...") - - packageRelativePath := generateCtx.TypeSpecConfig.GetPackageRelativePath() - if packageRelativePath == "" { - return fmt.Errorf("package module relative path not found") - } - - moduleRelativePath := generateCtx.TypeSpecConfig.GetModuleRelativePath() - if moduleRelativePath == "" { - return fmt.Errorf("module relative path not found in %s", generateCtx.TypeSpecConfig.Path) - } - - if !strings.HasPrefix(packageRelativePath, moduleRelativePath) { - return fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath) - } - result, err = generateCtx.GenerateForTypeSpec(&common.GenerateParam{ RPName: c.rpName, NamespaceName: c.namespaceName, @@ -209,7 +194,7 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str GoVersion: c.flags.GoVersion, TypeSpecEmitOption: c.flags.TypeSpecGoOption, TspClientOptions: c.flags.TspClientOption, - }, packageRelativePath, moduleRelativePath) + }) } else { log.Printf("Generate SDK through AutoRest...") result, err = generateCtx.GenerateForSingleRPNamespace(&common.GenerateParam{ From 692c91d26f377b58cd36105f944c3ea693c12cd8 Mon Sep 17 00:00:00 2001 From: JiaqiZhang-Dev <jiaqzhang@microsoft.com> Date: Thu, 6 Feb 2025 15:14:36 +0800 Subject: [PATCH 5/7] refactor variable name --- eng/tools/generator/cmd/v2/common/generation.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/tools/generator/cmd/v2/common/generation.go b/eng/tools/generator/cmd/v2/common/generation.go index 442426cc4c89..b7373842b2f6 100644 --- a/eng/tools/generator/cmd/v2/common/generation.go +++ b/eng/tools/generator/cmd/v2/common/generation.go @@ -377,7 +377,7 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate } func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (*GenerateResult, error) { - isModule := true + isSubPackage := false packageRelativePath := ctx.TypeSpecConfig.GetPackageRelativePath() if packageRelativePath == "" { @@ -387,7 +387,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* moduleRelativePath := ctx.TypeSpecConfig.GetModuleRelativePath() // if module relative path is not provided, find it from the sdk path by go.mod if moduleRelativePath == "" { - isModule = false + isSubPackage = true val, err := FindModuleDirByGoMod(filepath.Join(ctx.SDKPath, packageRelativePath)) if err != nil { return nil, err @@ -404,7 +404,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* } if packageRelativePath != moduleRelativePath { - isModule = false + isSubPackage = true } // if rp name and namespace name are not provided, extract them from the module path @@ -467,7 +467,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* log.Printf("Start to run `tsp-client init` to generate the code...") defaultModuleVersion := version.String() emitOption := "" - if isModule { + if !isSubPackage { emitOption = fmt.Sprintf("module-version=%s", defaultModuleVersion) } if generateParam.TypeSpecEmitOption != "" { @@ -564,7 +564,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* return nil, err } - if isModule { + if isSubPackage { // remove go.mod for sub package goModPath := filepath.Join(packagePath, "go.mod") if _, err := os.Stat(goModPath); !os.IsNotExist(err) { From 686ed83de5b96c62bbba040c6e096a2e910dce6a Mon Sep 17 00:00:00 2001 From: JiaqiZhang-Dev <jiaqzhang@microsoft.com> Date: Thu, 6 Feb 2025 15:26:43 +0800 Subject: [PATCH 6/7] update generate param --- eng/tools/generator/cmd/v2/common/generation.go | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/tools/generator/cmd/v2/common/generation.go b/eng/tools/generator/cmd/v2/common/generation.go index b7373842b2f6..6678a09dcd74 100644 --- a/eng/tools/generator/cmd/v2/common/generation.go +++ b/eng/tools/generator/cmd/v2/common/generation.go @@ -56,7 +56,6 @@ type GenerateParam struct { ForceStableVersion bool TypeSpecEmitOption string TspClientOptions []string - IsModule bool } func (ctx *GenerateContext) GenerateForAutomation(readme, repo, goVersion string) ([]GenerateResult, []error) { From a98be4c62e35e650568d3ba051bef05ec6efd3bf Mon Sep 17 00:00:00 2001 From: JiaqiZhang-Dev <jiaqzhang@microsoft.com> Date: Mon, 10 Feb 2025 15:00:26 +0800 Subject: [PATCH 7/7] refine find go.mod logic --- eng/tools/generator/cmd/v2/common/constants.go | 2 +- eng/tools/generator/cmd/v2/common/fileProcessor.go | 14 +++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/eng/tools/generator/cmd/v2/common/constants.go b/eng/tools/generator/cmd/v2/common/constants.go index d6dbcf314b93..cff5decdd1f5 100644 --- a/eng/tools/generator/cmd/v2/common/constants.go +++ b/eng/tools/generator/cmd/v2/common/constants.go @@ -6,5 +6,5 @@ package common const ( ChangelogFileName = "CHANGELOG.md" GoModFileName = "go.mod" - SdkRootPath = "sdk" + SdkRootPath = "/sdk" ) diff --git a/eng/tools/generator/cmd/v2/common/fileProcessor.go b/eng/tools/generator/cmd/v2/common/fileProcessor.go index c9c703e7c2c6..7a58a799b4d6 100644 --- a/eng/tools/generator/cmd/v2/common/fileProcessor.go +++ b/eng/tools/generator/cmd/v2/common/fileProcessor.go @@ -818,17 +818,9 @@ func FindModuleDirByGoMod(root string) (string, error) { curLevel++ continue } - entries, err := os.ReadDir(path) - if err != nil { - return "", err - } - for _, entry := range entries { - if entry.IsDir() { - continue - } - if entry.Name() == GoModFileName { - return path, nil - } + goModFilePath := filepath.Join(path, GoModFileName) + if _, err := os.Stat(goModFilePath); err == nil { + return path, nil } path = filepath.Dir(path) curLevel++