Google Ads source: support for custom WHERE clause for GAQL query

jitsucom · Sep 8, 2023 · 73a52ca · 73a52ca
1 parent f42f387
commit 73a52ca
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 7 deletions.
diff --git a/configurator/frontend/catalog/src/sources/lib/native.tsx b/configurator/frontend/catalog/src/sources/lib/native.tsx
@@ -207,7 +207,14 @@ export const googleAds: SourceConnector = {
       type: isoUtcDateType,
       defaultValue: "2020-01-01",
       required: true,
-    },
+    },{
+      displayName: "Custom 'WHERE' clause",
+      documentation: <>(Advanced) Custom GAQL query 'WHERE' clause to filter data.<br/>For example:<br/><code>{`campaign.start_date >= $DATE_FROM AND campaign.start_date <= $DATE_TO`}</code></>,
+      id: "where",
+      // prettier-ignore
+      type: stringType,
+      required: false,
+    }
   ],
   collectionTemplates: [
     {

diff --git a/server/drivers/google_ads/config.go b/server/drivers/google_ads/config.go
@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-//googleAdsHTTPConfiguration contains default amplitude HTTP timeouts/retry/delays,etc
+// googleAdsHTTPConfiguration contains default amplitude HTTP timeouts/retry/delays,etc
 var googleAdsHTTPConfiguration = &adapters.HTTPConfiguration{
 	GlobalClientTimeout:       10 * time.Minute,
 	RetryDelay:                10 * time.Second,
@@ -29,6 +29,7 @@ type GoogleAdsConfig struct {
 type GoogleAdsCollectionConfig struct {
 	StartDateStr string `mapstructure:"start_date" json:"start_date,omitempty" yaml:"start_date,omitempty"`
 	Fields       string `mapstructure:"fields" json:"fields,omitempty" yaml:"fields,omitempty"`
+	Where        string `mapstructure:"where" json:"where,omitempty" yaml:"where,omitempty"`
 }
 
 func (gac *GoogleAdsConfig) FillPreconfiguredOauth(sourceType string) {

diff --git a/server/drivers/google_ads/google_ads.go b/server/drivers/google_ads/google_ads.go
@@ -25,6 +25,8 @@ const (
 	dayLayout       = "2006-01-02"
 	dateLayoutFull  = "2006-01-02 15:04:05"
 	serviceEndpoint = "https://googleads.googleapis.com/v13"
+	dateFromMacro   = "$date_from"
+	dateToMacro     = "$date_to"
 )
 
 //go:embed reports.csv
@@ -59,6 +61,7 @@ type GoogleAds struct {
 
 	collection  *base.Collection
 	config      *GoogleAdsConfig
+	where       string
 	fields      []string
 	httpClient  *http.Client
 	granularity schema.Granularity
@@ -103,7 +106,8 @@ func NewGoogleAds(ctx context.Context, sourceConfig *base.SourceConfig, collecti
 	if !availableReports[collection.Type] {
 		return nil, fmt.Errorf("Unknown collection [%s]", collection.Type)
 	}
-
+	where := reportConfig.Where
+	where = strings.ReplaceAll(strings.ToLower(where), "where", "")
 	fields := strings.Split(strings.ReplaceAll(reportConfig.Fields, " ", ""), ",")
 
 	granularity := schema.ALL
@@ -116,13 +120,18 @@ func NewGoogleAds(ctx context.Context, sourceConfig *base.SourceConfig, collecti
 		i := sort.SearchStrings(sortedFields, pair.name)
 		if i < len(sortedFields) && sortedFields[i] == pair.name {
 			granularity = pair.granularity
+			where = utils.JoinNonEmptyStrings(" AND ", fmt.Sprintf("segments.date BETWEEN %s AND %s", dateFromMacro, dateToMacro), where)
 			break
 		}
 	}
+	if granularity == schema.ALL && where != "" && (strings.Contains(where, dateFromMacro) || strings.Contains(where, dateToMacro)) {
+		granularity = schema.DAY
+	}
 	return &GoogleAds{
 		IntervalDriver: base.IntervalDriver{SourceType: sourceConfig.Type},
 		collection:     collection,
 		config:         config,
+		where:          where,
 		fields:         fields,
 		httpClient:     httpClient,
 		granularity:    granularity,
@@ -163,8 +172,11 @@ func (g *GoogleAds) GetAllAvailableIntervals() ([]*base.TimeInterval, error) {
 
 func (g *GoogleAds) GetObjectsFor(interval *base.TimeInterval, objectsLoader base.ObjectsLoader) error {
 	gaql := "SELECT " + strings.Join(g.fields, ",") + " FROM " + g.collection.Type
-	if !interval.IsAll() {
-		gaql += fmt.Sprintf(" WHERE segments.date BETWEEN '%s' AND '%s'", interval.LowerEndpoint().Format(dayLayout), interval.UpperEndpoint().Format(dayLayout))
+
+	if g.where != "" {
+		where := strings.ReplaceAll(g.where, dateFromMacro, fmt.Sprintf("'%s'", interval.LowerEndpoint().Format(dayLayout)))
+		where = strings.ReplaceAll(where, dateToMacro, fmt.Sprintf("'%s'", interval.UpperEndpoint().Format(dayLayout)))
+		gaql += " WHERE " + where
 	}
 	loaded := 0
 	pageToken := ""

diff --git a/server/utils/strings.go b/server/utils/strings.go
@@ -1,8 +1,10 @@
 package utils
 
-//NvlString returns first not empty string value from varargs
+import "strings"
+
+// NvlString returns first not empty string value from varargs
 //
-//return "" if all strings are empty
+// return "" if all strings are empty
 func NvlString(args ...string) string {
 	for _, str := range args {
 		if str != "" {
@@ -27,3 +29,29 @@ func ShortenStringWithEllipsis(str string, n int) string {
 	}
 	return str[:n] + "..."
 }
+
+// JoinNonEmptyStrings joins strings with separator, but ignoring empty strings
+func JoinNonEmptyStrings(sep string, elems ...string) string {
+	switch len(elems) {
+	case 0:
+		return ""
+	case 1:
+		return elems[0]
+	}
+	n := len(sep) * (len(elems) - 1)
+	for i := 0; i < len(elems); i++ {
+		n += len(elems[i])
+	}
+
+	var b strings.Builder
+	b.Grow(n)
+	for _, s := range elems {
+		if len(s) > 0 {
+			if b.Len() > 0 {
+				b.WriteString(sep)
+			}
+			b.WriteString(s)
+		}
+	}
+	return b.String()
+}