diff --git a/eng/tools/generator/cmd/v2/automation/automationCmd.go b/eng/tools/generator/cmd/v2/automation/automationCmd.go index 1a4ae5c17dfe..14578e6a051f 100644 --- a/eng/tools/generator/cmd/v2/automation/automationCmd.go +++ b/eng/tools/generator/cmd/v2/automation/automationCmd.go @@ -115,69 +115,57 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline continue } - if ok := tsc.ExistEmitOption(string(typespec.TypeSpec_GO)); ok { - log.Printf("Start to process typespec project: %s", tspProjectFolder) - generateCtx := common.GenerateContext{ - SDKPath: sdkRepo.Root(), - SDKRepo: &sdkRepo, - SpecPath: ctx.specRoot, - SpecCommitHash: ctx.commitHash, - SpecRepoURL: input.RepoHTTPSURL, - TypeSpecConfig: tsc, - } - - module, err := tsc.GetModuleName() - if err != nil { - errorBuilder.add(err) - continue - } - packageModuleRelativePath := tsc.GetPackageModuleRelativePath() - if packageModuleRelativePath == "" { - errorBuilder.add(fmt.Errorf("package module relative path not found in %s", tspconfigPath)) - continue - } - namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{ - RPName: module[0], - NamespaceName: module[1], - SkipGenerateExample: true, - GoVersion: ctx.goVersion, - TspClientOptions: []string{"--debug"}, - }, packageModuleRelativePath) - if err != nil { - errorBuilder.add(err) - continue - } else { - content := namespaceResult.ChangelogMD - breaking := namespaceResult.Changelog.HasBreakingChanges() - breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems() - - srcFolder := filepath.Join(sdkRepo.Root(), packageModuleRelativePath) - apiViewArtifact := filepath.Join(sdkRepo.Root(), packageModuleRelativePath+".gosource") - err := zipDirectory(srcFolder, apiViewArtifact) - if err != nil { - fmt.Println(err) - } - - results = append(results, pipeline.PackageResult{ - Version: namespaceResult.Version, - PackageName: packageModuleRelativePath, - Path: []string{packageModuleRelativePath}, - PackageFolder: packageModuleRelativePath, - TypespecProject: []string{tspProjectFolder}, - Changelog: &pipeline.Changelog{ - Content: &content, - HasBreakingChange: &breaking, - BreakingChangeItems: &breakingChangeItems, - }, - APIViewArtifact: packageModuleRelativePath + ".gosource", - Language: "Go", - }) - - log.Printf("Finish processing typespec file: %s", tspconfigPath) - } - } else { + if ok := tsc.ExistEmitOption(string(typespec.TypeSpec_GO)); !ok { errorBuilder.add(fmt.Errorf("`@azure-tools/typespec-go` option not found in %s, it is required, please refer to `https://aka.ms/azsdk/tspconfig-sample-mpg` to configure it", tspconfigPath)) + continue + } + log.Printf("Start to process typespec project: %s", tspProjectFolder) + generateCtx := common.GenerateContext{ + SDKPath: sdkRepo.Root(), + SDKRepo: &sdkRepo, + SpecPath: ctx.specRoot, + SpecCommitHash: ctx.commitHash, + SpecRepoURL: input.RepoHTTPSURL, + TypeSpecConfig: tsc, } + + namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{ + SkipGenerateExample: true, + GoVersion: ctx.goVersion, + TspClientOptions: []string{"--debug"}, + }) + if err != nil { + errorBuilder.add(err) + continue + } + content := namespaceResult.ChangelogMD + breaking := namespaceResult.Changelog.HasBreakingChanges() + breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems() + packageRelativePath := namespaceResult.PackageRelativePath + + srcFolder := filepath.Join(sdkRepo.Root(), packageRelativePath) + apiViewArtifact := filepath.Join(sdkRepo.Root(), packageRelativePath+".gosource") + err = zipDirectory(srcFolder, apiViewArtifact) + if err != nil { + fmt.Println(err) + } + + results = append(results, pipeline.PackageResult{ + Version: namespaceResult.Version, + PackageName: packageRelativePath, + Path: []string{packageRelativePath}, + PackageFolder: packageRelativePath, + TypespecProject: []string{tspProjectFolder}, + Changelog: &pipeline.Changelog{ + Content: &content, + HasBreakingChange: &breaking, + BreakingChangeItems: &breakingChangeItems, + }, + APIViewArtifact: packageRelativePath + ".gosource", + Language: "Go", + }) + + log.Printf("Finish processing typespec file: %s", tspconfigPath) } // autorest diff --git a/eng/tools/generator/cmd/v2/common/changelogProcessor.go b/eng/tools/generator/cmd/v2/common/changelogProcessor.go index b3bd46e78834..8060c6d4da54 100644 --- a/eng/tools/generator/cmd/v2/common/changelogProcessor.go +++ b/eng/tools/generator/cmd/v2/common/changelogProcessor.go @@ -30,8 +30,8 @@ const ( sdk_remote_url = "https://github.com/Azure/azure-sdk-for-go.git" ) -func GetAllVersionTags(packageModuleRelativePath string) ([]string, error) { - arr := strings.Split(packageModuleRelativePath, "/") +func GetAllVersionTags(moduleRelativePath string) ([]string, error) { + arr := strings.Split(moduleRelativePath, "/") log.Printf("Fetching all release tags from GitHub for RP: '%s' Package: '%s' ...", arr[len(arr)-2], arr[len(arr)-1]) client := http.Client{} res, err := client.Get(sdk_tag_fetch_url) @@ -52,7 +52,7 @@ func GetAllVersionTags(packageModuleRelativePath string) ([]string, error) { versionTag := make(map[string]string) for _, tag := range result { tagName := tag["ref"].(string) - if strings.Contains(tagName, packageModuleRelativePath+"/v") { + if strings.Contains(tagName, moduleRelativePath+"/v") { m := regexp.MustCompile(semver.SemVerRegex).FindString(tagName) versions = append(versions, m) versionTag[m] = tagName @@ -168,7 +168,8 @@ func GetExportsFromTag(sdkRepo repo.SDKRepository, packagePath, tag string) (*ex // get exports result, err := exports.Get(packagePath) - if err != nil { + // bypass the error if the package doesn't contain any exports, return nil + if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") { return nil, err } diff --git a/eng/tools/generator/cmd/v2/common/constants.go b/eng/tools/generator/cmd/v2/common/constants.go index 836c12d3662b..d6dbcf314b93 100644 --- a/eng/tools/generator/cmd/v2/common/constants.go +++ b/eng/tools/generator/cmd/v2/common/constants.go @@ -5,4 +5,6 @@ package common const ( ChangelogFileName = "CHANGELOG.md" + GoModFileName = "go.mod" + SdkRootPath = "sdk" ) diff --git a/eng/tools/generator/cmd/v2/common/fileProcessor.go b/eng/tools/generator/cmd/v2/common/fileProcessor.go index 5cfb5f692da1..c9c703e7c2c6 100644 --- a/eng/tools/generator/cmd/v2/common/fileProcessor.go +++ b/eng/tools/generator/cmd/v2/common/fileProcessor.go @@ -138,6 +138,9 @@ func ReadV2ModuleNameToGetNamespace(path string) (map[string][]PackageInfo, erro // remove all sdk generated files in given path func CleanSDKGeneratedFiles(path string) error { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } log.Printf("Removing all sdk generated files in '%s'...", path) return filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { if err != nil { @@ -397,7 +400,6 @@ func AddChangelogToFile(changelog *Changelog, version *semver.Version, packageRo // replace `{{NewClientName}}` placeholder in README.md by first func name according to `^New.+Method$` pattern func ReplaceNewClientNamePlaceholder(packageRootPath string, exports exports.Content) error { - path := filepath.Join(packageRootPath, "README.md") var clientName string for _, k := range SortFuncItem(exports.Funcs) { v := exports.Funcs[k] @@ -407,6 +409,12 @@ func ReplaceNewClientNamePlaceholder(packageRootPath string, exports exports.Con } } + path := filepath.Join(packageRootPath, "README.md") + + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + b, err := os.ReadFile(path) if err != nil { return fmt.Errorf("cannot read from file '%s': %+v", path, err) @@ -420,9 +428,13 @@ func UpdateModuleDefinition(packageRootPath, packageModuleRelativePath string, v if version.Major() > 1 { path := filepath.Join(packageRootPath, "go.mod") + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + b, err := os.ReadFile(path) if err != nil { - return fmt.Errorf("cannot parse version from changelog") + return fmt.Errorf("cannot read go.mod") } lines := strings.Split(string(b), "\n") @@ -691,6 +703,11 @@ func ReplaceReadmeNewClientName(packageRootPath string, exports exports.Content) func ReplaceConstModuleVersion(packagePath string, newVersion string) error { path := filepath.Join(packagePath, "constants.go") + + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + data, err := os.ReadFile(path) if err != nil { return err @@ -789,3 +806,32 @@ func importPath(s *ast.ImportSpec) string { } return t } + +// Walks the sdk directory to find module based on a go.mod file +func FindModuleDirByGoMod(root string) (string, error) { + path := root + curLevel := 0 + maxLevel := 5 + for !strings.HasSuffix(path, SdkRootPath) && curLevel < maxLevel { + if _, err := os.Stat(path); os.IsNotExist(err) { + path = filepath.Dir(path) + curLevel++ + continue + } + entries, err := os.ReadDir(path) + if err != nil { + return "", err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + if entry.Name() == GoModFileName { + return path, nil + } + } + path = filepath.Dir(path) + curLevel++ + } + return "", fmt.Errorf("not found module, package path:%s", root) +} diff --git a/eng/tools/generator/cmd/v2/common/fileProcessor_test.go b/eng/tools/generator/cmd/v2/common/fileProcessor_test.go index 63219d2274f3..cdb981a48575 100644 --- a/eng/tools/generator/cmd/v2/common/fileProcessor_test.go +++ b/eng/tools/generator/cmd/v2/common/fileProcessor_test.go @@ -4,11 +4,16 @@ package common import ( + "fmt" + "os" + "path/filepath" "testing" + "github.com/Azure/azure-sdk-for-go/eng/tools/generator/repo" "github.com/Azure/azure-sdk-for-go/eng/tools/internal/delta" "github.com/Azure/azure-sdk-for-go/eng/tools/internal/exports" "github.com/Azure/azure-sdk-for-go/eng/tools/internal/report" + "github.com/Azure/azure-sdk-for-go/eng/tools/internal/utils" "github.com/stretchr/testify/assert" ) @@ -122,3 +127,16 @@ func TestCalculateNewVersion(t *testing.T) { assert.Equal(t, newVersion.String(), "1.2.0-beta.2") assert.Equal(t, BetaLabel, prl) } + +func TestFindModule(t *testing.T) { + cwd, err := os.Getwd() + assert.NoError(t, err) + sdkRoot := utils.NormalizePath(cwd) + sdkRepo, err := repo.OpenSDKRepository(sdkRoot) + assert.NoError(t, err) + module, err := FindModuleDirByGoMod(fmt.Sprintf("%s/%s", filepath.ToSlash(sdkRepo.Root()), "sdk/security/keyvault/azadmin/settings")) + assert.NoError(t, err) + moduleRelativePath, err := filepath.Rel(sdkRepo.Root(), module) + assert.NoError(t, err) + assert.Equal(t, "sdk/security/keyvault/azadmin", filepath.ToSlash(moduleRelativePath)) +} diff --git a/eng/tools/generator/cmd/v2/common/generation.go b/eng/tools/generator/cmd/v2/common/generation.go index 3e229cb3d00e..6678a09dcd74 100644 --- a/eng/tools/generator/cmd/v2/common/generation.go +++ b/eng/tools/generator/cmd/v2/common/generation.go @@ -32,13 +32,14 @@ type GenerateContext struct { } type GenerateResult struct { - Version string - RPName string - PackageName string - PackageAbsPath string - Changelog Changelog - ChangelogMD string - PullRequestLabels string + Version string + RPName string + PackageName string + PackageAbsPath string + Changelog Changelog + ChangelogMD string + PullRequestLabels string + PackageRelativePath string } type GenerateParam struct { @@ -235,7 +236,7 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate log.Printf("Start to generate changelog for package...") newExports, err := exports.Get(packagePath) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") { return nil, err } changelog, err := GetChangelogForPackage(oriExports, &newExports) @@ -374,9 +375,50 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate } } -func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, packageModuleRelativePath string) (*GenerateResult, error) { - packagePath := filepath.Join(ctx.SDKPath, packageModuleRelativePath) - changelogPath := filepath.Join(packagePath, ChangelogFileName) +func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (*GenerateResult, error) { + isSubPackage := false + + packageRelativePath := ctx.TypeSpecConfig.GetPackageRelativePath() + if packageRelativePath == "" { + return nil, fmt.Errorf("package module relative path not found in %s", ctx.TypeSpecConfig.Path) + } + + moduleRelativePath := ctx.TypeSpecConfig.GetModuleRelativePath() + // if module relative path is not provided, find it from the sdk path by go.mod + if moduleRelativePath == "" { + isSubPackage = true + val, err := FindModuleDirByGoMod(filepath.Join(ctx.SDKPath, packageRelativePath)) + if err != nil { + return nil, err + } + moduleRelativePath, err = filepath.Rel(ctx.SDKPath, val) + if err != nil { + return nil, err + } + moduleRelativePath = filepath.ToSlash(moduleRelativePath) + } + + if !strings.HasPrefix(packageRelativePath, moduleRelativePath) { + return nil, fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath) + } + + if packageRelativePath != moduleRelativePath { + isSubPackage = true + } + + // if rp name and namespace name are not provided, extract them from the module path + if len(generateParam.RPName) == 0 && len(generateParam.NamespaceName) == 0 { + rpAndNamespaceName, err := ctx.TypeSpecConfig.GetRpAndPackageNameByModule(moduleRelativePath) + if err != nil { + return nil, err + } + generateParam.RPName = rpAndNamespaceName[0] + generateParam.NamespaceName = rpAndNamespaceName[1] + } + + packagePath := filepath.Join(ctx.SDKPath, packageRelativePath) + modulePath := filepath.Join(ctx.SDKPath, moduleRelativePath) + changelogPath := filepath.Join(modulePath, ChangelogFileName) version, err := semver.NewVersion("0.1.0") if err != nil { @@ -393,21 +435,23 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa onBoard := false if _, err := os.Stat(changelogPath); os.IsNotExist(err) { onBoard = true - log.Printf("Package '%s' changelog not exist, do onboard process", packagePath) + log.Printf("Module '%s' changelog not exist, do onboard process", modulePath) if generateParam.SpecificPackageTitle == "" { generateParam.SpecificPackageTitle = strings.Title(generateParam.RPName) } log.Printf("Start to use template to generate new rp folder and basic package files...") sdkBasicInfo := map[string]any{ - "rpName": generateParam.RPName, - "packageName": generateParam.NamespaceName, - "packageTitle": generateParam.SpecificPackageTitle, - "packageVersion": version.String(), - "releaseDate": generateParam.ReleaseDate, - "goVersion": generateParam.GoVersion, - } - err = typespec.ParseTypeSpecTemplates(filepath.Join(ctx.SDKPath, "eng/tools/generator/template/typespec"), packagePath, sdkBasicInfo, nil) + "rpName": generateParam.RPName, + "packageName": generateParam.NamespaceName, + "moduleRelativePath": moduleRelativePath, + "serviceDir": strings.Replace(moduleRelativePath, "sdk/", "", 1), + "packageTitle": generateParam.SpecificPackageTitle, + "packageVersion": version.String(), + "releaseDate": generateParam.ReleaseDate, + "goVersion": generateParam.GoVersion, + } + err = typespec.ParseTypeSpecTemplates(filepath.Join(ctx.SDKPath, "eng/tools/generator/template/typespec"), modulePath, sdkBasicInfo, nil) if err != nil { return nil, err } @@ -421,7 +465,10 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa log.Printf("Start to run `tsp-client init` to generate the code...") defaultModuleVersion := version.String() - emitOption := fmt.Sprintf("module-version=%s", defaultModuleVersion) + emitOption := "" + if !isSubPackage { + emitOption = fmt.Sprintf("module-version=%s", defaultModuleVersion) + } if generateParam.TypeSpecEmitOption != "" { emitOption = fmt.Sprintf("%s;%s", emitOption, generateParam.TypeSpecEmitOption) } @@ -448,13 +495,13 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa if !onBoard { log.Printf("Get ori exports for changelog generation...") - tags, err := GetAllVersionTags(packageModuleRelativePath) + tags, err := GetAllVersionTags(moduleRelativePath) if err != nil { return nil, err } if len(tags) == 0 { - return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/%s hasn't been released, it's supposed to OnBoard", packageModuleRelativePath) + return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/%s hasn't been released, it's supposed to OnBoard", packageRelativePath) } previousVersionTag := GetPreviousVersionTag(isCurrentPreview, tags) @@ -470,7 +517,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa log.Printf("Start to generate changelog for package...") newExports, err := exports.Get(packagePath) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") { return nil, err } changelog, err := GetChangelogForPackage(oriExports, &newExports) @@ -484,7 +531,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa var prl PullRequestLabel if onBoard { log.Printf("Replace {{NewClientName}} placeholder in the README.md ") - if err = ReplaceNewClientNamePlaceholder(packagePath, newExports); err != nil { + if err = ReplaceNewClientNamePlaceholder(modulePath, newExports); err != nil { return nil, err } @@ -500,7 +547,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } log.Printf("Replace version in CHANGELOG.md...") - if err = UpdateOnboardChangelogVersion(packagePath, version.String()); err != nil { + if err = UpdateOnboardChangelogVersion(modulePath, version.String()); err != nil { return nil, err } @@ -516,19 +563,30 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - log.Printf("##[command]Executing go mod tidy in %s\n", packagePath) - if err = ExecuteGo(packagePath, "mod", "tidy"); err != nil { + if isSubPackage { + // remove go.mod for sub package + goModPath := filepath.Join(packagePath, "go.mod") + if _, err := os.Stat(goModPath); !os.IsNotExist(err) { + if err = os.Remove(goModPath); err != nil { + return nil, err + } + } + } + + log.Printf("##[command]Executing go mod tidy in %s\n", modulePath) + if err = ExecuteGo(modulePath, "mod", "tidy"); err != nil { return nil, err } return &GenerateResult{ - Version: version.String(), - RPName: generateParam.RPName, - PackageName: generateParam.NamespaceName, - PackageAbsPath: packagePath, - Changelog: *changelog, - ChangelogMD: changelog.ToCompactMarkdown() + "\n" + changelog.GetChangeSummary(), - PullRequestLabels: string(prl), + Version: version.String(), + RPName: generateParam.RPName, + PackageName: generateParam.NamespaceName, + PackageAbsPath: packagePath, + Changelog: *changelog, + ChangelogMD: changelog.ToCompactMarkdown() + "\n" + changelog.GetChangeSummary(), + PullRequestLabels: string(prl), + PackageRelativePath: packageRelativePath, }, nil } else { log.Printf("Calculate new version...") @@ -540,13 +598,13 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } log.Printf("Add changelog to file...") - changelogMd, err := AddChangelogToFile(changelog, version, packagePath, generateParam.ReleaseDate) + changelogMd, err := AddChangelogToFile(changelog, version, modulePath, generateParam.ReleaseDate) if err != nil { return nil, err } log.Printf("Update module definition if v2+...") - err = UpdateModuleDefinition(packagePath, packageModuleRelativePath, version) + err = UpdateModuleDefinition(packagePath, packageRelativePath, version) if err != nil { return nil, err } @@ -561,7 +619,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - baseModule := fmt.Sprintf("%s/%s", "github.com/Azure/azure-sdk-for-go", packageModuleRelativePath) + baseModule := fmt.Sprintf("%s/%s", "github.com/Azure/azure-sdk-for-go", packageRelativePath) if _, err := os.Stat(filepath.Join(packagePath, "fake")); !os.IsNotExist(err) && oldModuleVersion.Major() != version.Major() { log.Printf("Replace fake module v2+...") if err = ReplaceModule(version, packagePath, baseModule, ".go"); err != nil { @@ -578,12 +636,12 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa } log.Printf("Replace README.md module...") - if err = replaceReadmeModule(packagePath, packageModuleRelativePath, version.String()); err != nil { + if err = replaceReadmeModule(modulePath, packageRelativePath, version.String()); err != nil { return nil, err } log.Printf("Replace README.md NewClient name...") - if err = ReplaceReadmeNewClientName(packagePath, newExports); err != nil { + if err = ReplaceReadmeNewClientName(modulePath, newExports); err != nil { return nil, err } @@ -614,19 +672,20 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa return nil, err } - log.Printf("##[command]Executing go mod tidy in %s\n", packagePath) - if err = ExecuteGo(packagePath, "mod", "tidy"); err != nil { + log.Printf("##[command]Executing go mod tidy in %s\n", modulePath) + if err = ExecuteGo(modulePath, "mod", "tidy"); err != nil { return nil, err } return &GenerateResult{ - Version: version.String(), - RPName: generateParam.RPName, - PackageName: generateParam.NamespaceName, - PackageAbsPath: packagePath, - Changelog: *changelog, - ChangelogMD: changelogMd + "\n" + changelog.GetChangeSummary(), - PullRequestLabels: string(prl), + Version: version.String(), + RPName: generateParam.RPName, + PackageName: generateParam.NamespaceName, + PackageAbsPath: packagePath, + Changelog: *changelog, + ChangelogMD: changelogMd + "\n" + changelog.GetChangeSummary(), + PullRequestLabels: string(prl), + PackageRelativePath: packageRelativePath, }, nil } } diff --git a/eng/tools/generator/cmd/v2/release/releaseCmd.go b/eng/tools/generator/cmd/v2/release/releaseCmd.go index ceb0d3141420..42e0a866c223 100644 --- a/eng/tools/generator/cmd/v2/release/releaseCmd.go +++ b/eng/tools/generator/cmd/v2/release/releaseCmd.go @@ -183,10 +183,6 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str if existTypeSpec { log.Printf("Generate SDK through TypeSpec...") - packageModuleRelativePath := generateCtx.TypeSpecConfig.GetPackageModuleRelativePath() - if packageModuleRelativePath == "" { - return fmt.Errorf("package module relative path not found") - } result, err = generateCtx.GenerateForTypeSpec(&common.GenerateParam{ RPName: c.rpName, NamespaceName: c.namespaceName, @@ -198,7 +194,7 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str GoVersion: c.flags.GoVersion, TypeSpecEmitOption: c.flags.TypeSpecGoOption, TspClientOptions: c.flags.TspClientOption, - }, packageModuleRelativePath) + }) } else { log.Printf("Generate SDK through AutoRest...") result, err = generateCtx.GenerateForSingleRPNamespace(&common.GenerateParam{ diff --git a/eng/tools/generator/config/typespecRequests.go b/eng/tools/generator/config/typespecRequests.go index e78671a30556..c268b8ad284f 100644 --- a/eng/tools/generator/config/typespecRequests.go +++ b/eng/tools/generator/config/typespecRequests.go @@ -48,7 +48,7 @@ func GetTypeSpecProjectsFromConfig(config *Config, specRoot string) (tspProjects if err != nil { return nil, err } - module, err := tspConfig.GetModuleName() + module, err := tspConfig.GetRpAndPackageName() if err != nil { return nil, err } diff --git a/eng/tools/generator/config/validate/localValidator.go b/eng/tools/generator/config/validate/localValidator.go index 9318e5cd34d4..75562855bc71 100644 --- a/eng/tools/generator/config/validate/localValidator.go +++ b/eng/tools/generator/config/validate/localValidator.go @@ -82,7 +82,7 @@ func getReadmeGoFromReadme(readme string) string { return strings.ReplaceAll(readme, readmeFilename, goReadmeFilename) } -func GetModuleName(content []byte) (string, string) { +func GetRpAndPackageName(content []byte) (string, string) { moduleExist := regexp.MustCompile(goReadmeModuleName).Match(content) if moduleExist { moduleName := regexp.MustCompile(goReadmeModuleName).FindString(string(content)) diff --git a/eng/tools/generator/template/typespec/CHANGELOG.md.tpl b/eng/tools/generator/template/typespec/CHANGELOG.md.tpl index 07ec06e679bc..cbe14a38128a 100644 --- a/eng/tools/generator/template/typespec/CHANGELOG.md.tpl +++ b/eng/tools/generator/template/typespec/CHANGELOG.md.tpl @@ -3,6 +3,6 @@ ## {{.packageVersion}} ({{.releaseDate}}) ### Other Changes -The package of `github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{{.rpName}}/{{.packageName}}` is using our [next generation design principles](https://azure.github.io/azure-sdk/general_introduction.html). +The package of `github.com/Azure/azure-sdk-for-go/sdk/{{.moduleRelativePath}}` is using our [next generation design principles](https://azure.github.io/azure-sdk/general_introduction.html). To learn more, please refer to our documentation [Quick Start](https://aka.ms/azsdk/go/mgmt). \ No newline at end of file diff --git a/eng/tools/generator/template/typespec/README.md.tpl b/eng/tools/generator/template/typespec/README.md.tpl index 1b8d1bb0f8b1..66bee1288771 100644 --- a/eng/tools/generator/template/typespec/README.md.tpl +++ b/eng/tools/generator/template/typespec/README.md.tpl @@ -2,7 +2,7 @@ The `{{.packageName}}` module provides operations for working with Azure {{.packageTitle}}. -[Source code](https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/resourcemanager/{{.rpName}}/{{.packageName}}) +[Source code](https://github.com/Azure/azure-sdk-for-go/tree/main/{{.moduleRelativePath}}) # Getting started @@ -18,7 +18,7 @@ This project uses [Go modules](https://github.com/golang/go/wiki/Modules) for ve Install the Azure {{.packageTitle}} module: ```sh -go get github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{{.rpName}}/{{.packageName}} +go get github.com/Azure/azure-sdk-for-go/{{.moduleRelativePath}} ``` ## Authorization diff --git a/eng/tools/generator/template/typespec/ci.yml.tpl b/eng/tools/generator/template/typespec/ci.yml.tpl index b699dc9a3e14..f5ec43fee3d0 100644 --- a/eng/tools/generator/template/typespec/ci.yml.tpl +++ b/eng/tools/generator/template/typespec/ci.yml.tpl @@ -8,7 +8,7 @@ trigger: - release/* paths: include: - - sdk/resourcemanager/{{.rpName}}/{{.packageName}}/ + - {{.moduleRelativePath}}/ pr: branches: @@ -19,10 +19,10 @@ pr: - release/* paths: include: - - sdk/resourcemanager/{{.rpName}}/{{.packageName}}/ + - {{.moduleRelativePath}}/ extends: template: /eng/pipelines/templates/jobs/archetype-sdk-client.yml parameters: IncludeRelease: true - ServiceDirectory: 'resourcemanager/{{.rpName}}/{{.packageName}}' + ServiceDirectory: '{{.serviceDir}}' diff --git a/eng/tools/generator/template/typespec/go.mod.tpl b/eng/tools/generator/template/typespec/go.mod.tpl index 39e1daf4f726..117db716062b 100644 --- a/eng/tools/generator/template/typespec/go.mod.tpl +++ b/eng/tools/generator/template/typespec/go.mod.tpl @@ -1,4 +1,4 @@ -module github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{{.rpName}}/{{.packageName}} +module github.com/Azure/azure-sdk-for-go/{{.moduleRelativePath}} go {{.goVersion}} diff --git a/eng/tools/generator/typespec/tspconfig.go b/eng/tools/generator/typespec/tspconfig.go index bf402eb24678..671181e970d3 100644 --- a/eng/tools/generator/typespec/tspconfig.go +++ b/eng/tools/generator/typespec/tspconfig.go @@ -89,9 +89,10 @@ func ParseTypeSpecConfig(tspconfigPath string) (*TypeSpecConfig, error) { } goOption := emitOption.(map[string]any) - module, ok := goOption["module"].(string) - if !ok { - return nil, fmt.Errorf("the module must be set in %s option", TypeSpec_GO) + module, moduleOK := goOption["module"].(string) + _, packageOK := goOption["package-dir"].(string) + if !moduleOK && !packageOK { + return nil, fmt.Errorf("the module or package must be set in %s option", TypeSpec_GO) } if strings.Contains(module, "{service-dir}") { @@ -117,7 +118,19 @@ func ParseTypeSpecConfig(tspconfigPath string) (*TypeSpecConfig, error) { return &tspConfig, err } -func (tc *TypeSpecConfig) GetPackageModuleRelativePath() string { +func (tc *TypeSpecConfig) GetPackageRelativePath() string { + goConfig := tc.Options["@azure-tools/typespec-go"].(map[string]interface{}) + if goConfig["package-dir"] == nil { + return tc.GetModuleRelativePath() + } else { + if goConfig["service-dir"] == nil { + goConfig["service-dir"] = tc.Parameters["service-dir"].(map[string]interface{})["default"] + } + return goConfig["service-dir"].(string) + "/" + goConfig["package-dir"].(string) + } +} + +func (tc *TypeSpecConfig) GetModuleRelativePath() string { goConfig := tc.Options["@azure-tools/typespec-go"].(map[string]interface{}) if goConfig["module"] == nil { return "" @@ -180,19 +193,28 @@ func (tc TypeSpecConfig) ExistEmitOption(emit string) bool { return err == nil } -// GetModuleName return [rpName, packageName] -// module: github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{rpName}/{packageName} -func (tc TypeSpecConfig) GetModuleName() ([2]string, error) { +// GetRpAndPackageName return [rpName, packageName] +// module: github.com/Azure/azure-sdk-for-go/sdk/.../{rpName}/{packageName} +func (tc TypeSpecConfig) GetRpAndPackageName() ([2]string, error) { option, err := tc.EmitOption(string(TypeSpec_GO)) if err != nil { return [2]string{}, err } + goOption := option.(map[string]any) + module, ok := goOption["module"].(string) + if !ok || len(module) == 0 { + return [2]string{}, nil + } + return tc.GetRpAndPackageNameByModule(module) +} - module := (option.(map[string]any))["module"].(string) +// GetRpAndPackageName return [rpName, packageName] +// module: github.com/Azure/azure-sdk-for-go/sdk/.../{rpName}/{packageName} +func (tc TypeSpecConfig) GetRpAndPackageNameByModule(module string) ([2]string, error) { s := strings.Split(module, "/") l := len(s) - if l != 7 { - return [2]string{}, fmt.Errorf("module is invalid and must be in the format of `github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{rpName}/{packageName}`") + if l < 2 { + return [2]string{}, fmt.Errorf("module is invalid") } if !strings.Contains(s[l-1], "arm") && !strings.Contains(s[l-1], "az") { return [2]string{}, fmt.Errorf("packageName is invalid and must start with `arm` or `az`") diff --git a/eng/tools/generator/typespec/tspconfig_test.go b/eng/tools/generator/typespec/tspconfig_test.go index 5800b3355f44..332b322c5058 100644 --- a/eng/tools/generator/typespec/tspconfig_test.go +++ b/eng/tools/generator/typespec/tspconfig_test.go @@ -7,14 +7,14 @@ import ( "github.com/stretchr/testify/assert" ) -func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { +func TestTypeSpecConfig_GetPackageRelativePath(t *testing.T) { tests := []struct { name string config typespec.TypeSpecConfig expected string }{ { - name: "Valid module path", + name: "Package path from module", config: typespec.TypeSpecConfig{ TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ Options: map[string]any{ @@ -27,7 +27,75 @@ func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { expected: "sdk/messaging/eventgrid/azsystemevents", }, { - name: "Module path with placeholders", + name: "Package path from module with placeholder", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/{service-dir}/armcompute", + "service-dir": "sdk/resourcemanager/compute", + }, + }, + }, + }, + expected: "sdk/resourcemanager/compute/armcompute", + }, + { + name: "Empty package path", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{}, + }, + }, + }, + expected: "", + }, + { + name: "Package path from service and package dir", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/{service-dir}/azadmin", + "service-dir": "sdk/security/keyvault", + "package-dir": "azadmin/backup", + }, + }, + }, + }, + expected: "sdk/security/keyvault/azadmin/backup", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.config.GetPackageRelativePath()) + }) + } +} + +func TestTypeSpecConfig_GetModuleRelativePath(t *testing.T) { + tests := []struct { + name string + config typespec.TypeSpecConfig + expected string + }{ + { + name: "Normal", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/sdk/messaging/eventgrid/azsystemevents", + }, + }, + }, + }, + expected: "sdk/messaging/eventgrid/azsystemevents", + }, + { + name: "Module with placeholder", config: typespec.TypeSpecConfig{ TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ Options: map[string]any{ @@ -42,7 +110,7 @@ func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { expected: "sdk/resourcemanager/compute/armcompute", }, { - name: "Module path without module key", + name: "Empty module", config: typespec.TypeSpecConfig{ TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ Options: map[string]any{ @@ -52,11 +120,26 @@ func TestTypeSpecConfig_GetPackageModuleRelativePath(t *testing.T) { }, expected: "", }, + { + name: "Module different from package path", + config: typespec.TypeSpecConfig{ + TypeSpecProjectSchema: typespec.TypeSpecProjectSchema{ + Options: map[string]any{ + "@azure-tools/typespec-go": map[string]any{ + "module": "github.com/Azure/azure-sdk-for-go/{service-dir}/azadmin", + "service-dir": "sdk/security/keyvault", + "package-dir": "azadmin/backup", + }, + }, + }, + }, + expected: "sdk/security/keyvault/azadmin", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, tt.config.GetPackageModuleRelativePath()) + assert.Equal(t, tt.expected, tt.config.GetModuleRelativePath()) }) } } diff --git a/eng/tools/generator/typespec/typespec-go.go b/eng/tools/generator/typespec/typespec-go.go index a311728d0a8e..8c0fe99a7e28 100644 --- a/eng/tools/generator/typespec/typespec-go.go +++ b/eng/tools/generator/typespec/typespec-go.go @@ -2,6 +2,7 @@ package typespec import ( "errors" + "log" "regexp" "github.com/goccy/go-yaml" @@ -56,7 +57,8 @@ var ( func (o *GoEmitterOptions) Validate() error { if o.Module == "" { - return ErrModuleEmpty + log.Printf("typesepec-go option `module` is empty") + return nil } matched := regexp.MustCompile(moduleRegex).MatchString(o.Module)