Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refine sdk automation tool #23959

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

refine sdk automation tool #23959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d0cbce
bypass empty export error
tadelesh Jan 14, 2025
ecc6a7c
support sub package
tadelesh Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 26 additions & 13 deletions eng/tools/generator/cmd/v2/automation/automationCmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,24 +125,37 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline
SpecRepoURL: input.RepoHTTPSURL,
TypeSpecConfig: tsc,
}

module, err := tsc.GetModuleName()
rpAndNamespaceName, err := tsc.GetRpAndPackageName()
if err != nil {
errorBuilder.add(err)
continue
}
packageModuleRelativePath := tsc.GetPackageModuleRelativePath()
if packageModuleRelativePath == "" {

packageRelativePath := tsc.GetPackageRelativePath()
if packageRelativePath == "" {
errorBuilder.add(fmt.Errorf("package module relative path not found in %s", tspconfigPath))
continue
}

moduleRelativePath := tsc.GetModuleRelativePath()
if moduleRelativePath == "" {
errorBuilder.add(fmt.Errorf("module relative path not found in %s", tspconfigPath))
continue
}

if !strings.HasPrefix(packageRelativePath, moduleRelativePath) {
errorBuilder.add(fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath))
continue
}

namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{
RPName: module[0],
NamespaceName: module[1],
RPName: rpAndNamespaceName[0],
NamespaceName: rpAndNamespaceName[1],
SkipGenerateExample: true,
GoVersion: ctx.goVersion,
TspClientOptions: []string{"--debug"},
}, packageModuleRelativePath)
}, packageRelativePath, moduleRelativePath)
if err != nil {
errorBuilder.add(err)
continue
Expand All @@ -151,25 +164,25 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline
breaking := namespaceResult.Changelog.HasBreakingChanges()
breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems()

srcFolder := filepath.Join(sdkRepo.Root(), packageModuleRelativePath)
apiViewArtifact := filepath.Join(sdkRepo.Root(), packageModuleRelativePath+".gosource")
srcFolder := filepath.Join(sdkRepo.Root(), packageRelativePath)
apiViewArtifact := filepath.Join(sdkRepo.Root(), packageRelativePath+".gosource")
err := zipDirectory(srcFolder, apiViewArtifact)
if err != nil {
fmt.Println(err)
}

results = append(results, pipeline.PackageResult{
Version: namespaceResult.Version,
PackageName: packageModuleRelativePath,
Path: []string{packageModuleRelativePath},
PackageFolder: packageModuleRelativePath,
PackageName: packageRelativePath,
Path: []string{packageRelativePath},
PackageFolder: packageRelativePath,
TypespecProject: []string{tspProjectFolder},
Changelog: &pipeline.Changelog{
Content: &content,
HasBreakingChange: &breaking,
BreakingChangeItems: &breakingChangeItems,
},
APIViewArtifact: packageModuleRelativePath + ".gosource",
APIViewArtifact: packageRelativePath + ".gosource",
Language: "Go",
})

Expand Down
9 changes: 5 additions & 4 deletions eng/tools/generator/cmd/v2/common/changelogProcessor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ const (
sdk_remote_url = "https://github.com/Azure/azure-sdk-for-go.git"
)

func GetAllVersionTags(packageModuleRelativePath string) ([]string, error) {
arr := strings.Split(packageModuleRelativePath, "/")
func GetAllVersionTags(moduleRelativePath string) ([]string, error) {
arr := strings.Split(moduleRelativePath, "/")
log.Printf("Fetching all release tags from GitHub for RP: '%s' Package: '%s' ...", arr[len(arr)-2], arr[len(arr)-1])
client := http.Client{}
res, err := client.Get(sdk_tag_fetch_url)
Expand All @@ -52,7 +52,7 @@ func GetAllVersionTags(packageModuleRelativePath string) ([]string, error) {
versionTag := make(map[string]string)
for _, tag := range result {
tagName := tag["ref"].(string)
if strings.Contains(tagName, packageModuleRelativePath+"/v") {
if strings.Contains(tagName, moduleRelativePath+"/v") {
m := regexp.MustCompile(semver.SemVerRegex).FindString(tagName)
versions = append(versions, m)
versionTag[m] = tagName
Expand Down Expand Up @@ -168,7 +168,8 @@ func GetExportsFromTag(sdkRepo repo.SDKRepository, packagePath, tag string) (*ex

// get exports
result, err := exports.Get(packagePath)
if err != nil {
// bypass the error if the package doesn't contain any exports, return nil
if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") {
return nil, err
}

Expand Down
18 changes: 16 additions & 2 deletions eng/tools/generator/cmd/v2/common/fileProcessor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,6 @@ func AddChangelogToFile(changelog *Changelog, version *semver.Version, packageRo

// replace `{{NewClientName}}` placeholder in README.md by first func name according to `^New.+Method$` pattern
func ReplaceNewClientNamePlaceholder(packageRootPath string, exports exports.Content) error {
path := filepath.Join(packageRootPath, "README.md")
var clientName string
for _, k := range SortFuncItem(exports.Funcs) {
v := exports.Funcs[k]
Expand All @@ -407,6 +406,12 @@ func ReplaceNewClientNamePlaceholder(packageRootPath string, exports exports.Con
}
}

path := filepath.Join(packageRootPath, "README.md")

if _, err := os.Stat(path); os.IsNotExist(err) {
return nil
}

b, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("cannot read from file '%s': %+v", path, err)
Expand All @@ -420,9 +425,13 @@ func UpdateModuleDefinition(packageRootPath, packageModuleRelativePath string, v
if version.Major() > 1 {
path := filepath.Join(packageRootPath, "go.mod")

if _, err := os.Stat(path); os.IsNotExist(err) {
return nil
}

b, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("cannot parse version from changelog")
return fmt.Errorf("cannot read go.mod")
}

lines := strings.Split(string(b), "\n")
Expand Down Expand Up @@ -691,6 +700,11 @@ func ReplaceReadmeNewClientName(packageRootPath string, exports exports.Content)

func ReplaceConstModuleVersion(packagePath string, newVersion string) error {
path := filepath.Join(packagePath, "constants.go")

if _, err := os.Stat(path); os.IsNotExist(err) {
return nil
}

data, err := os.ReadFile(path)
if err != nil {
return err
Expand Down
67 changes: 40 additions & 27 deletions eng/tools/generator/cmd/v2/common/generation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate

log.Printf("Start to generate changelog for package...")
newExports, err := exports.Get(packagePath)
if err != nil {
if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") {
return nil, err
}
changelog, err := GetChangelogForPackage(oriExports, &newExports)
Expand Down Expand Up @@ -374,9 +374,10 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate
}
}

func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, packageModuleRelativePath string) (*GenerateResult, error) {
packagePath := filepath.Join(ctx.SDKPath, packageModuleRelativePath)
changelogPath := filepath.Join(packagePath, ChangelogFileName)
func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, packageRelativePath string, moduleRelativePath string) (*GenerateResult, error) {
packagePath := filepath.Join(ctx.SDKPath, packageRelativePath)
modulePath := filepath.Join(ctx.SDKPath, moduleRelativePath)
changelogPath := filepath.Join(modulePath, ChangelogFileName)

version, err := semver.NewVersion("0.1.0")
if err != nil {
Expand All @@ -393,21 +394,23 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
onBoard := false
if _, err := os.Stat(changelogPath); os.IsNotExist(err) {
onBoard = true
log.Printf("Package '%s' changelog not exist, do onboard process", packagePath)
log.Printf("Module '%s' changelog not exist, do onboard process", modulePath)
if generateParam.SpecificPackageTitle == "" {
generateParam.SpecificPackageTitle = strings.Title(generateParam.RPName)
}

log.Printf("Start to use template to generate new rp folder and basic package files...")
sdkBasicInfo := map[string]any{
"rpName": generateParam.RPName,
"packageName": generateParam.NamespaceName,
"packageTitle": generateParam.SpecificPackageTitle,
"packageVersion": version.String(),
"releaseDate": generateParam.ReleaseDate,
"goVersion": generateParam.GoVersion,
}
err = typespec.ParseTypeSpecTemplates(filepath.Join(ctx.SDKPath, "eng/tools/generator/template/typespec"), packagePath, sdkBasicInfo, nil)
"rpName": generateParam.RPName,
"packageName": generateParam.NamespaceName,
"moduleRelativePath": moduleRelativePath,
"serviceDir": strings.Replace(moduleRelativePath, "sdk/", "", 1),
"packageTitle": generateParam.SpecificPackageTitle,
"packageVersion": version.String(),
"releaseDate": generateParam.ReleaseDate,
"goVersion": generateParam.GoVersion,
}
err = typespec.ParseTypeSpecTemplates(filepath.Join(ctx.SDKPath, "eng/tools/generator/template/typespec"), modulePath, sdkBasicInfo, nil)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -448,13 +451,13 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
if !onBoard {
log.Printf("Get ori exports for changelog generation...")

tags, err := GetAllVersionTags(packageModuleRelativePath)
tags, err := GetAllVersionTags(moduleRelativePath)
if err != nil {
return nil, err
}

if len(tags) == 0 {
return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/%s hasn't been released, it's supposed to OnBoard", packageModuleRelativePath)
return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/%s hasn't been released, it's supposed to OnBoard", packageRelativePath)
}

previousVersionTag := GetPreviousVersionTag(isCurrentPreview, tags)
Expand All @@ -470,7 +473,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa

log.Printf("Start to generate changelog for package...")
newExports, err := exports.Get(packagePath)
if err != nil {
if err != nil && !strings.Contains(err.Error(), "doesn't contain any exports") {
return nil, err
}
changelog, err := GetChangelogForPackage(oriExports, &newExports)
Expand All @@ -484,7 +487,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
var prl PullRequestLabel
if onBoard {
log.Printf("Replace {{NewClientName}} placeholder in the README.md ")
if err = ReplaceNewClientNamePlaceholder(packagePath, newExports); err != nil {
if err = ReplaceNewClientNamePlaceholder(modulePath, newExports); err != nil {
return nil, err
}

Expand All @@ -500,7 +503,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
}

log.Printf("Replace version in CHANGELOG.md...")
if err = UpdateOnboardChangelogVersion(packagePath, version.String()); err != nil {
if err = UpdateOnboardChangelogVersion(modulePath, version.String()); err != nil {
return nil, err
}

Expand All @@ -516,8 +519,18 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
return nil, err
}

log.Printf("##[command]Executing go mod tidy in %s\n", packagePath)
if err = ExecuteGo(packagePath, "mod", "tidy"); err != nil {
if packageRelativePath != moduleRelativePath {
// remove go.mod for sub package
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are some questions about this operation

  1. The condition of existing sub package is like this?
    service-dir: "sdk/resourcemanager/service_name"
    package-dir: "package_name"
    module: "github.com/Azure/azure-sdk-for-go/{service-dir}/{package-dir}/subpackage_name"
  2. why was the sub package's go.mod generated? Is it a problem of 'tsp-client init' command?
  3. Does there has existed these sub package go.mod? how about processing the current existed go.mod for sub package in the repo?

goModPath := filepath.Join(packagePath, "go.mod")
if _, err := os.Stat(goModPath); !os.IsNotExist(err) {
if err = os.Remove(goModPath); err != nil {
return nil, err
}
}
}

log.Printf("##[command]Executing go mod tidy in %s\n", modulePath)
if err = ExecuteGo(modulePath, "mod", "tidy"); err != nil {
return nil, err
}

Expand All @@ -540,13 +553,13 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
}

log.Printf("Add changelog to file...")
changelogMd, err := AddChangelogToFile(changelog, version, packagePath, generateParam.ReleaseDate)
changelogMd, err := AddChangelogToFile(changelog, version, modulePath, generateParam.ReleaseDate)
if err != nil {
return nil, err
}

log.Printf("Update module definition if v2+...")
err = UpdateModuleDefinition(packagePath, packageModuleRelativePath, version)
err = UpdateModuleDefinition(packagePath, packageRelativePath, version)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there should be the modelPath and moduleRelativePath?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

current impl is kind of a workaround. i separated the module from package since current go code generator does not support namespace in typespec.

if err != nil {
return nil, err
}
Expand All @@ -561,7 +574,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
return nil, err
}

baseModule := fmt.Sprintf("%s/%s", "github.com/Azure/azure-sdk-for-go", packageModuleRelativePath)
baseModule := fmt.Sprintf("%s/%s", "github.com/Azure/azure-sdk-for-go", packageRelativePath)
if _, err := os.Stat(filepath.Join(packagePath, "fake")); !os.IsNotExist(err) && oldModuleVersion.Major() != version.Major() {
log.Printf("Replace fake module v2+...")
if err = ReplaceModule(version, packagePath, baseModule, ".go"); err != nil {
Expand All @@ -578,12 +591,12 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
}

log.Printf("Replace README.md module...")
if err = replaceReadmeModule(packagePath, packageModuleRelativePath, version.String()); err != nil {
if err = replaceReadmeModule(modulePath, packageRelativePath, version.String()); err != nil {
return nil, err
}

log.Printf("Replace README.md NewClient name...")
if err = ReplaceReadmeNewClientName(packagePath, newExports); err != nil {
if err = ReplaceReadmeNewClientName(modulePath, newExports); err != nil {
return nil, err
}

Expand Down Expand Up @@ -614,8 +627,8 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, pa
return nil, err
}

log.Printf("##[command]Executing go mod tidy in %s\n", packagePath)
if err = ExecuteGo(packagePath, "mod", "tidy"); err != nil {
log.Printf("##[command]Executing go mod tidy in %s\n", modulePath)
if err = ExecuteGo(modulePath, "mod", "tidy"); err != nil {
return nil, err
}

Expand Down
17 changes: 14 additions & 3 deletions eng/tools/generator/cmd/v2/release/releaseCmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,10 +183,21 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str

if existTypeSpec {
log.Printf("Generate SDK through TypeSpec...")
packageModuleRelativePath := generateCtx.TypeSpecConfig.GetPackageModuleRelativePath()
if packageModuleRelativePath == "" {

packageRelativePath := generateCtx.TypeSpecConfig.GetPackageRelativePath()
if packageRelativePath == "" {
return fmt.Errorf("package module relative path not found")
}

moduleRelativePath := generateCtx.TypeSpecConfig.GetModuleRelativePath()
if moduleRelativePath == "" {
return fmt.Errorf("module relative path not found in %s", generateCtx.TypeSpecConfig.Path)
}

if !strings.HasPrefix(packageRelativePath, moduleRelativePath) {
return fmt.Errorf("module relative path '%s' is not a prefix of package relative path '%s', please check your tspconfig.yaml file", moduleRelativePath, packageRelativePath)
}

result, err = generateCtx.GenerateForTypeSpec(&common.GenerateParam{
RPName: c.rpName,
NamespaceName: c.namespaceName,
Expand All @@ -198,7 +209,7 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str
GoVersion: c.flags.GoVersion,
TypeSpecEmitOption: c.flags.TypeSpecGoOption,
TspClientOptions: c.flags.TspClientOption,
}, packageModuleRelativePath)
}, packageRelativePath, moduleRelativePath)
} else {
log.Printf("Generate SDK through AutoRest...")
result, err = generateCtx.GenerateForSingleRPNamespace(&common.GenerateParam{
Expand Down
2 changes: 1 addition & 1 deletion eng/tools/generator/config/typespecRequests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func GetTypeSpecProjectsFromConfig(config *Config, specRoot string) (tspProjects
if err != nil {
return nil, err
}
module, err := tspConfig.GetModuleName()
module, err := tspConfig.GetRpAndPackageName()
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion eng/tools/generator/config/validate/localValidator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func getReadmeGoFromReadme(readme string) string {
return strings.ReplaceAll(readme, readmeFilename, goReadmeFilename)
}

func GetModuleName(content []byte) (string, string) {
func GetRpAndPackageName(content []byte) (string, string) {
moduleExist := regexp.MustCompile(goReadmeModuleName).Match(content)
if moduleExist {
moduleName := regexp.MustCompile(goReadmeModuleName).FindString(string(content))
Expand Down
2 changes: 1 addition & 1 deletion eng/tools/generator/template/typespec/CHANGELOG.md.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
## {{.packageVersion}} ({{.releaseDate}})
### Other Changes

The package of `github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{{.rpName}}/{{.packageName}}` is using our [next generation design principles](https://azure.github.io/azure-sdk/general_introduction.html).
The package of `github.com/Azure/azure-sdk-for-go/sdk/{{.moduleRelativePath}}` is using our [next generation design principles](https://azure.github.io/azure-sdk/general_introduction.html).

To learn more, please refer to our documentation [Quick Start](https://aka.ms/azsdk/go/mgmt).
Loading
Loading