Filtering Search Console keywords (#4077)

* Apply filters in search console request

* Remove dead code from search console modal

* Remove unimportant information from keyword modal

* Show invalid filters from search console

* Fix tests

* Add/Fix tests

* Fix typo

* Remove unused variable

* Fix typo

* Changelog entry

* Fix Credo

* Display impressions, CTR and position in keyword modal

* Undo change that should not have been committed

* Fix test

* Fix test

* filters -> search_console_filters

CHANGELOG.md

@@ -42,6 +42,7 @@ All notable changes to this project will be documented in this file.
 - Add Yesterday as an time range option in the dashboard
 - Add support for importing Google Analytics 4 data
 - Import custom events from Google Analytics 4
+- Ability to filter Search Console keywords by page, country and device plausible/analytics#4077
 - Add `DATA_DIR` env var for exports/imports plausible/analytics#4100
 
 ### Removed

assets/js/dashboard/stats/modals/exit-pages.js

@@ -4,7 +4,7 @@ import { withRouter } from 'react-router-dom'
 
 import Modal from './modal'
 import * as api from '../../api'
-import numberFormatter from '../../util/number-formatter'
+import numberFormatter, {percentageFormatter} from '../../util/number-formatter'
 import { parseQuery } from '../../query'
 import { trimURL } from '../../util/url'
 class ExitPagesModal extends React.Component {
@@ -34,14 +34,6 @@ class ExitPagesModal extends React.Component {
     this.setState({ loading: true, page: this.state.page + 1 }, this.loadPages.bind(this))
   }
 
-  formatPercentage(number) {
-    if (typeof (number) === 'number') {
-      return number + '%'
-    } else {
-      return '-'
-    }
-  }
-
   showConversionRate() {
     return !!this.state.query.filters.goal
   }
@@ -74,7 +66,7 @@ class ExitPagesModal extends React.Component {
         {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.total_visitors)}</td>}
         <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.visitors)}</td>
         {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.visits)}</td>}
-        {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatPercentage(page.exit_rate)}</td>}
+        {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{percentageFormatter(page.exit_rate)}</td>}
         {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.conversion_rate)}%</td>}
       </tr>
     )

assets/js/dashboard/stats/modals/google-keywords.js

@@ -3,7 +3,7 @@ import { Link, withRouter } from 'react-router-dom'
 
 import Modal from './modal'
 import * as api from '../../api'
-import numberFormatter from '../../util/number-formatter'
+import numberFormatter, { percentageFormatter } from '../../util/number-formatter'
 import {parseQuery} from '../../query'
 import RocketIcon from './rocket-icon'
 
@@ -17,49 +17,31 @@ class GoogleKeywordsModal extends React.Component {
   }
 
   componentDidMount() {
-    if (this.state.query.filters.goal) {
-      api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers/Google`, this.state.query, {limit: 100})
-        .then((res) => this.setState({
-          loading: false,
-          searchTerms: res.search_terms,
-          totalVisitors: res.total_visitors,
-          notConfigured: res.not_configured,
-          isOwner: res.is_owner
-        }))
-    } else {
-      api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100})
-        .then((res) => this.setState({
-          loading: false,
-          searchTerms: res.search_terms,
-          totalVisitors: res.total_visitors,
-          notConfigured: res.not_configured,
-          isOwner: res.is_owner
-        }))
-    }
+    api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100})
+      .then((res) => this.setState({
+        loading: false,
+        searchTerms: res.search_terms,
+        notConfigured: res.not_configured,
+        isOwner: res.is_owner
+      }))
   }
 
   renderTerm(term) {
     return (
       <React.Fragment key={term.name}>
-
         <tr className="text-sm dark:text-gray-200" key={term.name}>
-          <td className="p-2 truncate">{term.name}</td>
+          <td className="p-2">{term.name}</td>
           <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.visitors)}</td>
+          <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.impressions)}</td>
+          <td className="p-2 w-32 font-medium" align="right">{percentageFormatter(term.ctr)}</td>
+          <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.position)}</td>
         </tr>
       </React.Fragment>
     )
   }
 
   renderKeywords() {
-    if (this.state.query.filters.goal) {
-      return (
-        <div className="text-center text-gray-700 dark:text-gray-300 mt-6">
-          <RocketIcon />
-          <div className="text-lg">Sorry, we cannot show which keywords converted best for goal <b>{this.state.query.filters.goal}</b></div>
-          <div className="text-lg">Google does not share this information</div>
-        </div>
-      )
-    } else if (this.state.notConfigured) {
+    if (this.state.notConfigured) {
       if (this.state.isOwner) {
         return (
           <div className="text-center text-gray-700 dark:text-gray-300 mt-6">
@@ -84,7 +66,10 @@ class GoogleKeywordsModal extends React.Component {
           <thead>
             <tr>
               <th className="p-2 w-48 md:w-56 lg:w-1/3 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="left">Search Term</th>
-              <th className="p-2 w-32 lg:w-1/2 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Visitors</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Visitors</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Impressions</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">CTR</th>
+              <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Position</th>
             </tr>
           </thead>
           <tbody>
@@ -102,14 +87,6 @@ class GoogleKeywordsModal extends React.Component {
     }
   }
 
-  renderGoalText() {
-    if (this.state.query.filters.goal) {
-      return (
-        <h1 className="text-xl font-semibold text-gray-500 dark:text-gray-200 leading-none">completed {this.state.query.filters.goal}</h1>
-      )
-    }
-  }
-
   renderBody() {
     if (this.state.loading) {
       return (
@@ -122,10 +99,6 @@ class GoogleKeywordsModal extends React.Component {
 
           <div className="my-4 border-b border-gray-300 dark:border-gray-500"></div>
           <main className="modal__content">
-            <h1 className="text-xl font-semibold mb-0 leading-none dark:text-gray-200">
-              {this.state.totalVisitors} visitors from Google<br />
-            </h1>
-            {this.renderGoalText()}
             { this.renderKeywords() }
           </main>
         </React.Fragment>

assets/js/dashboard/stats/sources/search-terms.js

@@ -39,7 +39,8 @@ export default class SearchTerms extends React.Component {
         loading: false,
         searchTerms: res.search_terms || [],
         notConfigured: res.not_configured,
-        isAdmin: res.is_admin
+        isAdmin: res.is_admin,
+        unsupportedFilters: res.unsupported_filters
       })).catch((error) =>
         {
             this.setState({ loading: false, searchTerms: [], notConfigured: true, error: true, isAdmin: error.payload.is_admin })
@@ -68,21 +69,19 @@ export default class SearchTerms extends React.Component {
   }
 
   renderList() {
-    if (this.props.query.filters.goal) {
+   if (this.state.unsupportedFilters)  {
       return (
         <div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20">
           <RocketIcon />
-          <div>Sorry, we cannot show which keywords converted best for goal <b>{this.props.query.filters.goal}</b></div>
-          <div>Google does not share this information</div>
+          <div>Unable to fetch keyword data from Search Console because it does not support the current set of filters</div>
         </div>
       )
-
     } else if (this.state.notConfigured) {
       return (
         <div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20">
           <RocketIcon />
           <div>
-          This site is not connected to Search Console so we cannot show the search phrases.
+          This site is not connected to Search Console so we cannot show the search terms
           {this.state.isAdmin && this.state.error && <><br/><br/><p>Please click below to connect your Search Console account.</p></>}
           </div>
           {this.state.isAdmin && <a href={`/${encodeURIComponent(this.props.site.domain)}/settings/integrations`} className="button mt-4">Connect with Google</a> }
@@ -103,9 +102,8 @@ export default class SearchTerms extends React.Component {
       )
     } else {
       return (
-        <div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20">
-          <RocketIcon />
-          <div>No search terms were found for this period. Please adjust or extend your time range. Check <a href="https://plausible.io/docs/google-search-console-integration#i-dont-see-google-search-query-data-in-my-dashboard" target="_blank" rel="noreferrer" className="hover:underline text-indigo-700 dark:text-indigo-500">our documentation</a> for more details.</div>
+        <div className="text-center text-gray-700 dark:text-gray-300 ">
+          <div className="mt-44 mx-auto font-medium text-gray-500 dark:text-gray-400">No data yet</div>
         </div>
       )
     }

assets/js/dashboard/util/number-formatter.js

@@ -49,3 +49,11 @@ export function durationFormatter(duration) {
     return `${seconds}s`
   }
 }
+
+export function percentageFormatter(number) {
+  if (typeof (number) === 'number') {
+    return number + '%'
+  } else {
+    return '-'
+  }
+}

fixture/http_mocks/google_analytics_stats.json

@@ -4,7 +4,7 @@
     "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",
     "method": "post",
     "request_body": {
-      "dimensionFilterGroups": {},
+      "dimensionFilterGroups": [],
       "dimensions": [
         "query"
       ],
@@ -16,26 +16,20 @@
       "responseAggregationType": "auto",
       "rows": [
         {
-          "clicks": 25.0,
-          "ctr": 0.3,
-          "impressions": 50.0,
-          "keys": [
-            "keyword1",
-            "keyword2"
-          ],
-          "position": 2.0
+          "clicks": 25,
+          "ctr": 0.3679,
+          "impressions": 50,
+          "keys": ["keyword1"],
+          "position": 2.2312312
         },
         {
-          "clicks": 15.0,
+          "clicks": 15,
           "ctr": 0.5,
-          "impressions": 25.0,
-          "keys": [
-            "keyword3",
-            "keyword4"
-          ],
+          "impressions": 25,
+          "keys": ["keyword3"],
           "position": 4.0
         }
       ]
     }
   }
-]
+]

lib/plausible/google/api.ex

@@ -6,6 +6,7 @@ defmodule Plausible.Google.API do
   use Timex
 
   alias Plausible.Google.HTTP
+  alias Plausible.Google.SearchConsole
 
   require Logger
 
@@ -74,21 +75,26 @@ defmodule Plausible.Google.API do
   end
 
   def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
-    with site <- Plausible.Repo.preload(site, :google_auth),
+    with {:ok, site} <- ensure_search_console_property(site),
          {:ok, access_token} <- maybe_refresh_token(site.google_auth),
+         {:ok, search_console_filters} <-
+           SearchConsole.Filters.transform(site.google_auth.property, filters),
          {:ok, stats} <-
            HTTP.list_stats(
              access_token,
              site.google_auth.property,
              date_range,
              limit,
-             filters["page"]
+             search_console_filters
            ) do
       stats
       |> Map.get("rows", [])
-      |> Enum.filter(fn row -> row["clicks"] > 0 end)
-      |> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end)
+      |> Enum.map(&search_console_row/1)
       |> then(&{:ok, &1})
+    else
+      :google_property_not_configured -> {:error, :google_property_not_configured}
+      :unsupported_filters -> {:error, :unsupported_filters}
+      {:error, error} -> {:error, error}
     end
   end
 
@@ -142,6 +148,44 @@ defmodule Plausible.Google.API do
     Timex.before?(expires_at, thirty_seconds_ago)
   end
 
+  defp ensure_search_console_property(site) do
+    site = Plausible.Repo.preload(site, :google_auth)
+
+    if site.google_auth && site.google_auth.property do
+      {:ok, site}
+    else
+      :google_property_not_configured
+    end
+  end
+
+  defp search_console_row(row) do
+    %{
+      # We always request just one dimension at a time (`query`)
+      name: row["keys"] |> List.first(),
+      visitors: row["clicks"],
+      impressions: row["impressions"],
+      ctr: rounded_ctr(row["ctr"]),
+      position: rounded_position(row["position"])
+    }
+  end
+
+  defp rounded_ctr(ctr) do
+    {:ok, decimal} = Decimal.cast(ctr)
+
+    decimal
+    |> Decimal.mult(100)
+    |> Decimal.round(1)
+    |> Decimal.to_float()
+  end
+
+  defp rounded_position(position) do
+    {:ok, decimal} = Decimal.cast(position)
+
+    decimal
+    |> Decimal.round(1)
+    |> Decimal.to_float()
+  end
+
   defp client_id() do
     Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
   end