Migrate keywords from Trac tickets to labels

README.md

@@ -7,12 +7,12 @@
 At present the following Trac data is converted:
 
 * Trac users mapped onto Gitea usernames (can be customised by providing an explicit mapping)
-* Trac components, priorities, resolutions, severities, types and versions to Gitea labels (can be customised by providing an explicit mapping)
+* Trac components, priorities, resolutions, severities, types, keywords and versions to Gitea labels (can be customised by providing an explicit mapping)
 * Trac milestones to Gitea milestones
 * Trac tickets to Gitea issues
   * Trac ticket attachments to Gitea issue attachments
   * Trac ticket comments to Gitea issue comments with markdown text conversion
-  * Trac ticket component, priority, resolution, severity, type and version changes to Gitea issue label changes
+  * Trac ticket component, priority, resolution, severity, type, keywords and version changes to Gitea issue label changes
   * Trac ticket milestone changes to Gitea issue milestone changes
   * Trac ticket owner changes to Gitea issue assignee changes
   * Trac ticket "close" and "reopen" status changes to Gitea issue equivalents

accessor/trac/accessor.go

@@ -39,6 +39,7 @@ type Ticket struct {
 	ResolutionName string
 	SeverityName   string
 	TypeName       string
+	Keywords       string
 	VersionName    string
 	Status         string
 	Created        int64
@@ -214,6 +215,14 @@ type Accessor interface {
 	// GetVersions retrieves all versions used in Trac, passing each one to the provided "handler" function.
 	GetVersions(handlerFn func(version *Label) error) error
 
+	/*
+	 * Keywords
+	 */
+	// ParseKeywords
+	ParseKeywords(keywords string) []string
+	// GetKeywords retrieves all keywords used in Trac tickets, passing each one to the provided "handler" function.
+	GetKeywords(handlerFn func(tracKeyword *Label) error) error
+
 	/*
 	 * Wiki
 	 */

accessor/trac/keywords.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Steve Jefferson. All rights reserved.
+// Use of this source code is governed by a GPL-style
+// license that can be found in the LICENSE file.
+
+package trac
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// ParseKeywords splits the Trac "keywords" string in a slice of separated keywords
+func (accessor *DefaultAccessor) ParseKeywords(keywords string) []string {
+	return strings.Fields(strings.ReplaceAll(keywords, ",", " "))
+}
+
+// GetKeywords retrieves all keywords used in Trac tickets, passing each one to the provided "handler" function.
+func (accessor *DefaultAccessor) GetKeywords(handlerFn func(tracKeyword *Label) error) error {
+	rows, err := accessor.db.Query(`SELECT DISTINCT COALESCE(keywords,'') keywords FROM ticket`)
+	if err != nil {
+		err = errors.Wrapf(err, "retrieving Trac keywords")
+		return err
+	}
+	// Parse each row for multiple keywords separates by coma or spaces
+	var keywords []string
+	for rows.Next() {
+		var rawKeywords string
+		if err := rows.Scan(&rawKeywords); err != nil {
+			err = errors.Wrapf(err, "retrieving Trac keywords")
+			return err
+		}
+		rowKeywords := accessor.ParseKeywords(rawKeywords)
+		// fmt.Println("Keywords:", rowKeywords)
+		for j := 0; j < len(rowKeywords); j++ {
+			if !slices.Contains(keywords, rowKeywords[j]) {
+				keywords = append(keywords, rowKeywords[j])
+			}
+		}
+	}
+
+	for i := 0; i < len(keywords); i++ {
+		keywordName := keywords[i]
+		tracKeyword := Label{Name: keywordName, Description: ""}
+		if err = handlerFn(&tracKeyword); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

accessor/trac/ticket.go

@@ -19,6 +19,7 @@ func (accessor *DefaultAccessor) GetTickets(handlerFn func(ticket *Ticket) error
 			COALESCE(t.priority,''),
 			COALESCE(t.owner,''),
 			t.reporter,
+			COALESCE(t.keywords,''),
 			COALESCE(t.version,''),
 			COALESCE(t.milestone,''),
 			lower(COALESCE(t.status, '')),
@@ -33,16 +34,16 @@ func (accessor *DefaultAccessor) GetTickets(handlerFn func(ticket *Ticket) error
 
 	for rows.Next() {
 		var ticketID, created, updated int64
-		var summary, description, owner, reporter, milestoneName, componentName, priorityName, resolutionName, severityName, typeName, versionName, status string
+		var summary, description, owner, reporter, milestoneName, componentName, priorityName, resolutionName, severityName, typeName, keywords, versionName, status string
 		if err := rows.Scan(&ticketID, &typeName, &created, &updated, &componentName, &severityName, &priorityName, &owner, &reporter,
-			&versionName, &milestoneName, &status, &resolutionName, &summary, &description); err != nil {
+			&keywords, &versionName, &milestoneName, &status, &resolutionName, &summary, &description); err != nil {
 			err = errors.Wrapf(err, "retrieving Trac ticket")
 			return err
 		}
 
 		ticket := Ticket{TicketID: ticketID, Summary: summary, Description: description, Owner: owner, Reporter: reporter,
 			MilestoneName: milestoneName, ComponentName: componentName, PriorityName: priorityName, ResolutionName: resolutionName,
-			SeverityName: severityName, TypeName: typeName, VersionName: versionName, Status: status, Created: created, Updated: updated}
+			SeverityName: severityName, TypeName: typeName, Keywords: keywords, VersionName: versionName, Status: status, Created: created, Updated: updated}
 
 		if err = handlerFn(&ticket); err != nil {
 			return err

importer/label.go

@@ -17,6 +17,7 @@ const (
 	resolutionLabelColor = "#9e9e9e"
 	severityLabelColor   = "#eb6420"
 	typeLabelColor       = "#e11d21"
+	keywordLabelColor    = "#ff00ff"
 	versionLabelColor    = "#009800"
 )
 
@@ -64,6 +65,11 @@ func (importer *Importer) DefaultTypeLabelMap() (map[string]string, error) {
 	return importer.defaultLabelMap(trac.Accessor.GetTypes)
 }
 
+// DefaultKeywordLabelMap retrieves the default mapping between Trac keywords and Gitea labels
+func (importer *Importer) DefaultKeywordLabelMap() (map[string]string, error) {
+	return importer.defaultLabelMap(trac.Accessor.GetKeywords)
+}
+
 // DefaultVersionLabelMap retrieves the default mapping between Trac versions and Gitea labels
 func (importer *Importer) DefaultVersionLabelMap() (map[string]string, error) {
 	return importer.defaultLabelMap(trac.Accessor.GetVersions)
@@ -152,6 +158,14 @@ func (importer *Importer) ImportTypes(typeNameMap map[string]string) error {
 	})
 }
 
+// ImportKeywords imports Trac keywords as Gitea labels.
+func (importer *Importer) ImportKeywords(keywordNameMap map[string]string) error {
+	return importer.tracAccessor.GetKeywords(func(keyword *trac.Label) error {
+		_, err := importer.importLabel(keyword, keywordNameMap, keywordLabelColor)
+		return err
+	})
+}
+
 // ImportVersions imports Trac versions as Gitea labels.
 func (importer *Importer) ImportVersions(versionNameMap map[string]string) error {
 	return importer.tracAccessor.GetVersions(func(version *trac.Label) error {

importer/setup_ticket_test.go

@@ -56,6 +56,7 @@ var (
 	resolutionMap map[string]string
 	severityMap   map[string]string
 	typeMap       map[string]string
+	keywordMap    map[string]string
 	versionMap    map[string]string
 	revisionMap   map[string]string
 )
@@ -66,6 +67,7 @@ func initMaps() {
 	resolutionMap = make(map[string]string)
 	severityMap = make(map[string]string)
 	typeMap = make(map[string]string)
+	keywordMap = make(map[string]string)
 	versionMap = make(map[string]string)
 }
 
@@ -102,6 +104,20 @@ var (
 	severityLabel2   *TicketLabelImport
 	typeLabel1       *TicketLabelImport
 	typeLabel2       *TicketLabelImport
+	keyword1         string
+	keyword2         string
+	keyword3         string
+	keywordLst1      string
+	keywordLst2      string
+	keywordLst3      string
+	keywordLst4      string
+	keywordLabel1    *TicketLabelImport
+	keywordLabel2    *TicketLabelImport
+	keywordLabel3    *TicketLabelImport
+	keywordLabelLst1 []*TicketLabelImport
+	keywordLabelLst2 []*TicketLabelImport
+	keywordLabelLst3 []*TicketLabelImport
+	keywordLabelLst4 []*TicketLabelImport
 	versionLabel1    *TicketLabelImport
 	versionLabel2    *TicketLabelImport
 )
@@ -117,6 +133,20 @@ func setUpTicketLabels(t *testing.T) {
 	severityLabel2 = createTicketLabelImport("severity2", severityMap)
 	typeLabel1 = createTicketLabelImport("type1", typeMap)
 	typeLabel2 = createTicketLabelImport("type2", typeMap)
+	keyword1 = "keyword1"
+	keyword2 = "keyword2"
+	keyword3 = "keyword3"
+	keywordLst1 = keyword1
+	keywordLst2 = keyword1 + " " + keyword2
+	keywordLst3 = keyword1 + ", " + keyword2 + "," + keyword3
+	keywordLst4 = ""
+	keywordLabel1 = createTicketLabelImport(keyword1, keywordMap)
+	keywordLabel2 = createTicketLabelImport(keyword2, keywordMap)
+	keywordLabel3 = createTicketLabelImport(keyword3, keywordMap)
+	keywordLabelLst1 = []*TicketLabelImport{keywordLabel1}
+	keywordLabelLst2 = []*TicketLabelImport{keywordLabel1, keywordLabel2}
+	keywordLabelLst3 = []*TicketLabelImport{keywordLabel1, keywordLabel2, keywordLabel3}
+	keywordLabelLst4 = nil
 	versionLabel1 = createTicketLabelImport("version1", versionMap)
 	versionLabel2 = createTicketLabelImport("version2", versionMap)
 }
@@ -136,6 +166,8 @@ type TicketImport struct {
 	resolutionLabel     *TicketLabelImport
 	severityLabel       *TicketLabelImport
 	typeLabel           *TicketLabelImport
+	keywords            string
+	keywordLabels       []*TicketLabelImport
 	versionLabel        *TicketLabelImport
 	closed              bool
 	status              string
@@ -153,6 +185,8 @@ func createTicketImport(
 	resolutionLabel *TicketLabelImport,
 	severityLabel *TicketLabelImport,
 	typeLabel *TicketLabelImport,
+	keywords string,
+	keywordLabels []*TicketLabelImport,
 	versionLabel *TicketLabelImport) *TicketImport {
 	status := "open"
 	if closed {
@@ -173,6 +207,8 @@ func createTicketImport(
 		resolutionLabel:     resolutionLabel,
 		severityLabel:       severityLabel,
 		typeLabel:           typeLabel,
+		keywords:            keywords,
+		keywordLabels:       keywordLabels,
 		versionLabel:        versionLabel,
 		closed:              closed,
 		status:              status,
@@ -194,6 +230,7 @@ func createTracTicket(ticket *TicketImport) *trac.Ticket {
 		ResolutionName: ticket.resolutionLabel.tracName,
 		SeverityName:   ticket.severityLabel.tracName,
 		TypeName:       ticket.typeLabel.tracName,
+		Keywords:       ticket.keywords,
 		VersionName:    ticket.versionLabel.tracName,
 		Status:         ticket.status,
 		Created:        ticket.created,
@@ -227,19 +264,19 @@ func setUpTickets(t *testing.T) {
 	closedTicket = createTicketImport(
 		"closed", true,
 		closedTicketOwner, closedTicketReporter,
-		componentLabel1, priorityLabel1, resolutionLabel1, severityLabel1, typeLabel1, versionLabel1)
+		componentLabel1, priorityLabel1, resolutionLabel1, severityLabel1, typeLabel1, keywordLst1, keywordLabelLst1, versionLabel1)
 	openTicket = createTicketImport(
 		"open", false,
 		openTicketOwner, openTicketReporter,
-		componentLabel2, priorityLabel2, resolutionLabel2, severityLabel2, typeLabel2, versionLabel2)
+		componentLabel2, priorityLabel2, resolutionLabel2, severityLabel2, typeLabel2, keywordLst2, keywordLabelLst2, versionLabel2)
 	noTracUserTicket = createTicketImport(
 		"noTracUser", false,
 		noTracUserTicketOwner, noTracUserTicketReporter,
-		componentLabel1, priorityLabel1, resolutionLabel1, severityLabel1, typeLabel1, versionLabel1)
+		componentLabel1, priorityLabel1, resolutionLabel1, severityLabel1, typeLabel1, keywordLst3, keywordLabelLst3, versionLabel1)
 	unmappedTracUserTicket = createTicketImport(
 		"unmappedTracUser", false,
 		unmappedTracUserTicketOwner, unmappedTracUserTicketReporter,
-		componentLabel1, priorityLabel1, resolutionLabel1, severityLabel1, typeLabel1, versionLabel1)
+		componentLabel1, priorityLabel1, resolutionLabel1, severityLabel1, typeLabel1, keywordLst4, keywordLabelLst4, versionLabel1)
 }
 
 func expectTracTicketRetrievals(t *testing.T, tickets ...*TicketImport) {
@@ -256,6 +293,22 @@ func expectTracTicketRetrievals(t *testing.T, tickets ...*TicketImport) {
 		})
 }
 
+func expectTracKeywordsParsing(t *testing.T, ticket *TicketImport, keywordLabels []*TicketLabelImport) {
+	// expect trac accessor to parse keywords in trac ticket
+	mockTracAccessor.
+		EXPECT().
+		ParseKeywords(gomock.Any()).
+		DoAndReturn(func(keywords string) []string {
+			assertEquals(t, keywords, ticket.keywords)
+			// Build the expected list of keywords from the labels
+			var keywordLst []string
+			for _, keywordLabel := range ticket.keywordLabels {
+				keywordLst = append(keywordLst, keywordLabel.tracName)
+			}
+			return keywordLst
+		})
+}
+
 func expectDescriptionMarkdownConversion(t *testing.T, ticket *TicketImport) {
 	mockMarkdownConverter.
 		EXPECT().
@@ -368,6 +421,10 @@ func expectAllTicketActions(t *testing.T, ticket *TicketImport) {
 	expectIssueLabelCreation(t, ticket, ticket.resolutionLabel)
 	expectIssueLabelCreation(t, ticket, ticket.severityLabel)
 	expectIssueLabelCreation(t, ticket, ticket.typeLabel)
+	expectTracKeywordsParsing(t, ticket, ticket.keywordLabels)
+	for _, keywordLabel := range ticket.keywordLabels {
+		expectIssueLabelCreation(t, ticket, keywordLabel)
+	}
 	expectIssueLabelCreation(t, ticket, ticket.versionLabel)
 
 	// expect the repo issue index to be updated

importer/ticket.go

@@ -64,7 +64,7 @@ func (importer *Importer) importTicket(ticket *trac.Ticket, closed bool, userMap
 
 // ImportTickets imports Trac tickets as Gitea issues.
 func (importer *Importer) ImportTickets(
-	userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap, revisionMap map[string]string) error {
+	userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, keywordMap, versionMap, revisionMap map[string]string) error {
 	err := importer.tracAccessor.GetTickets(func(ticket *trac.Ticket) error {
 		closed := (ticket.Status == string(trac.TicketStatusClosed))
 		issueID, err := importer.importTicket(ticket, closed, userMap, revisionMap)
@@ -100,6 +100,13 @@ func (importer *Importer) ImportTickets(
 			return err
 		}
 
+		for _, keywordName := range importer.tracAccessor.ParseKeywords(ticket.Keywords) {
+			_, err = importer.importTicketLabel(issueID, keywordName, keywordMap)
+			if err != nil {
+				return err
+			}
+		}
+
 		_, err = importer.importTicketLabel(issueID, ticket.VersionName, versionMap)
 		if err != nil {
 			return err
@@ -110,7 +117,7 @@ func (importer *Importer) ImportTickets(
 			return err
 		}
 		lastUpdate, err = importer.importTicketChanges(ticket.TicketID, issueID, lastUpdate,
-			userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap,
+			userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, keywordMap, versionMap,
 			revisionMap)
 		if err != nil {
 			return err

importer/ticketAttachment_test.go

@@ -41,7 +41,7 @@ func TestImportTicketWithAttachments(t *testing.T) {
 	// expect to update Gitea issue description
 	expectIssueDescriptionUpdates(t, closedTicket.issueID, closedTicket.descriptionMarkdown)
 
-	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap, revisionMap)
+	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, keywordMap, versionMap, revisionMap)
 }
 
 func TestImportMultipleTicketsWithAttachments(t *testing.T) {
@@ -88,7 +88,7 @@ func TestImportMultipleTicketsWithAttachments(t *testing.T) {
 	expectIssueDescriptionUpdates(t, closedTicket.issueID, closedTicket.descriptionMarkdown)
 	expectIssueDescriptionUpdates(t, openTicket.issueID, openTicket.descriptionMarkdown)
 
-	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap, revisionMap)
+	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, keywordMap, versionMap, revisionMap)
 }
 
 func TestImportTicketWithAttachmentButNoTracUser(t *testing.T) {
@@ -125,7 +125,7 @@ func TestImportTicketWithAttachmentButNoTracUser(t *testing.T) {
 	// expect to update Gitea issue description
 	expectIssueDescriptionUpdates(t, noTracUserTicket.issueID, noTracUserTicket.descriptionMarkdown)
 
-	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap, revisionMap)
+	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, keywordMap, versionMap, revisionMap)
 }
 
 func TestImportTicketWithAttachmentButUnmappedTracUser(t *testing.T) {
@@ -162,5 +162,5 @@ func TestImportTicketWithAttachmentButUnmappedTracUser(t *testing.T) {
 	// expect to update Gitea issue description
 	expectIssueDescriptionUpdates(t, unmappedTracUserTicket.issueID, unmappedTracUserTicket.descriptionMarkdown)
 
-	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap, revisionMap)
+	dataImporter.ImportTickets(userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, keywordMap, versionMap, revisionMap)
 }

importer/ticketChange.go

@@ -90,7 +90,7 @@ func (importer *Importer) importTicketChanges(
 	ticketID int64,
 	issueID int64,
 	lastUpdate int64,
-	userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap, revisionMap map[string]string) (int64, error) {
+	userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, keywordMap, versionMap, revisionMap map[string]string) (int64, error) {
 	commentLastUpdate := lastUpdate
 	err := importer.tracAccessor.GetTicketChanges(ticketID, func(change *trac.TicketChange) error {
 		commentID, err := importer.importTicketChange(issueID, change, userMap, componentMap, priorityMap, resolutionMap, severityMap, typeMap, versionMap, revisionMap)