knife fix when the rows in wild search exceeds 10k. (#3951)

Fix knife search issue with morethan 10K nodes.

Signed-off-by: talktovikas <vikas.yadav@progress.com>

Author: talktovikas

docs-chef-io/content/server/config_rb_server_optional_settings.md

@@ -2023,6 +2023,11 @@ This configuration file has the following settings for `opscode-erchef`:
 
     Default value: `127.0.0.1`.
 
+`opscode_erchef['track_total_hits']`
+
+:   Whether to return how many documents matched the query.
+
+    Default value: `false`.
 
 ### opscode-expander
 

oc-chef-pedant/spec/running_configs/basic_config_spec.rb

@@ -142,6 +142,10 @@
       expect(config['opscode-erchef']['solr_http_max_age'].to_s).not_to eq ''
     end
 
+    it "opscode-erchef/track_total_hits" do
+      expect(config['opscode-erchef']['track_total_hits'].to_s).not_to eq ''
+    end
+
     it "opscode-erchef/solr_http_max_connection_duration" do
       expect(config['opscode-erchef']['solr_http_max_connection_duration'].to_s).not_to eq ''
     end

omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb

@@ -344,6 +344,7 @@
 default['private_chef']['opscode-erchef']['search_batch_max_size'] = '5000000'
 default['private_chef']['opscode-erchef']['search_batch_max_wait'] = '10'
 default['private_chef']['opscode-erchef']['search_auth_username'] = 'opensearch_user'
+default['private_chef']['opscode-erchef']['track_total_hits'] = false
 # default['private_chef']['opscode-erchef']['search_auth_password'] = "admin"
 # solr_service configuration for erchef. These are used to configure an opscoderl_httpc pool
 # of HTTP connecton workers.

omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb

@@ -189,6 +189,7 @@
         {solr_elasticsearch_major_version, <%= @solr_elasticsearch_major_version %>},
         {search_auth_username, "<%= node['private_chef']['opscode-erchef']['search_auth_username'] %>"},
         {search_auth_password, "<%= @helper.search_auth_password() %>"},
+        {track_total_hits, <%= node['private_chef']['opscode-erchef']['track_total_hits'] -%>},
         {solr_service, [
             {root_url, "<%= @helper.search_engine_url() %>"},
             {timeout, <%= @solr_timeout %>},

src/oc_erchef/apps/chef_index/src/chef_index_query.erl

@@ -30,6 +30,7 @@ from_params(Provider, ObjType, QueryString, Start, Rows) ->
                      search_provider = Provider,
                      start = decode({nonneg_int, "start"}, Start, 0),
                      rows = decode({nonneg_int, "rows"}, Rows, 1000),
+                     track_total_hits = envy:get(chef_index, track_total_hits, false, boolean),
                      sort = "X_CHEF_id_CHEF_X asc",
                      index = index_type(ObjType)}.
 

src/oc_erchef/apps/chef_index/src/chef_opensearch.erl

@@ -103,21 +103,54 @@ query_body(#chef_solr_query{
               query_string = Query,
               filter_query = undefined,
               start = Start,
+              track_total_hits = true,
               rows = Rows}) ->
     jiffy:encode({[{fields_tag(), <<"_id">>},
                    {<<"from">>, Start},
                    {<<"size">>, Rows},
+                   {<<"track_total_hits">>,true},
                    {<<"query">>, {[query_string_query_ejson(Query)]}}
                   ]});
+query_body(#chef_solr_query{
+    query_string = Query,
+    filter_query = undefined,
+    start = Start,
+    track_total_hits = false,
+    rows = Rows}) ->
+    jiffy:encode({[{fields_tag(), <<"_id">>},
+                    {<<"from">>, Start},
+                    {<<"size">>, Rows},
+                    {<<"query">>, {[query_string_query_ejson(Query)]}}
+                ]});
+
+query_body(#chef_solr_query{
+    query_string = Query,
+    filter_query = FilterQuery,
+    start = Start,
+    track_total_hits = false,
+    rows = Rows}) ->
+chef_index_query:assert_org_id_filter(FilterQuery),
+jiffy:encode({[{ fields_tag(), <<"_id">>},
+    {<<"from">>, Start},
+    {<<"size">>, Rows},
+    {<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]},
+    {<<"query">>, {[
+        {<<"bool">>,{[
+            {<<"must">>, {[query_string_query_ejson(Query)]}},
+            {<<"filter">>, {[query_string_query_ejson(FilterQuery)]}}
+        ]}}]}
+    }]});
 query_body(#chef_solr_query{
               query_string = Query,
               filter_query = FilterQuery,
               start = Start,
+              track_total_hits = true,
               rows = Rows}) ->
     chef_index_query:assert_org_id_filter(FilterQuery),
     jiffy:encode({[{ fields_tag(), <<"_id">>},
         {<<"from">>, Start},
         {<<"size">>, Rows},
+        {<<"track_total_hits">>,true},
         {<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]},
         {<<"query">>, {[
             {<<"bool">>,{[

src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl

@@ -40,6 +40,7 @@ query_from_params_test_() ->
             sort = "X_CHEF_id_CHEF_X asc",
             start = 2,
             rows = 5,
+            track_total_hits=false,
             index = node},
           ?assertEqual(Expect, Query)
       end},
@@ -57,6 +58,7 @@ query_from_params_test_() ->
             sort = "X_CHEF_id_CHEF_X asc",
             start = 0,
             rows = 1000,
+            track_total_hits=false,
             index = role},
           ?assertEqual(Expect, Query)
       end},

src/oc_erchef/habitat/config/sys.config

@@ -173,6 +173,7 @@
         {search_batch_max_wait, 10},
         {reindex_sleep_min_ms, 500},
         {reindex_sleep_max_ms, 2000},
+        {track_total_hits, {{track_total_hits}} },
         {reindex_item_retries, 3},
         {solr_elasticsearch_major_version, 5},
         {solr_service, [

src/oc_erchef/habitat/default.toml

@@ -30,6 +30,7 @@ keygen_timeout=5000
 [chef_db]
 
 [chef_index]
+track_total_hits=false
 
 [chef_objects]
 

src/oc_erchef/include/chef_solr.hrl

@@ -21,6 +21,7 @@
           search_provider = solr :: 'solr' | 'elasticsearch' | 'opensearch',
           start :: integer() | undefined,
           rows :: integer()  | undefined,
+          track_total_hits :: boolean() |undefined,
           sort :: string()   | undefined,
           index :: 'node'
                  | 'role'