HIVE-29178: Add Catalog properties support

parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g

@@ -161,6 +161,14 @@ alterCatalogStatementSuffix
 @init { gParent.pushMsg("alter catalog statement", state); }
 @after { gParent.popMsg(state); }
     : alterCatalogSuffixSetLocation
+    | alterCatalogSuffixProperties
+    ;
+
+alterCatalogSuffixProperties
+@init { gParent.pushMsg("alter catalog properties statement", state); }
+@after { gParent.popMsg(state); }
+    : name=identifier KW_SET KW_PROPERTIES catProperties
+    -> ^(TOK_ALTERCATALOG_PROPERTIES $name catProperties)
     ;
 
 alterCatalogSuffixSetLocation

parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g

@@ -219,6 +219,7 @@ KW_VIEWS: 'VIEWS';
 KW_IN: 'IN';
 KW_CATALOG: 'CATALOG';
 KW_CATALOGS: 'CATALOGS';
+KW_PROPERTIES: 'PROPERTIES';
 KW_DATABASE: 'DATABASE';
 KW_DATABASES: 'DATABASES';
 KW_MATERIALIZED: 'MATERIALIZED';

parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g

@@ -159,6 +159,7 @@ TOK_UNIONTYPE;
 TOK_VARIANT;
 TOK_COLTYPELIST;
 TOK_CREATECATALOG;
+TOK_CATALOGPROPERTIES;
 TOK_CREATEDATABASE;
 TOK_CREATEDATACONNECTOR;
 TOK_CREATETABLE;
@@ -378,6 +379,7 @@ TOK_DESCCATALOG;
 TOK_CATALOGLOCATION;
 TOK_CATALOGCOMMENT;
 TOK_ALTERCATALOG_LOCATION;
+TOK_ALTERCATALOG_PROPERTIES;
 TOK_DESCDATABASE;
 TOK_DATABASEPROPERTIES;
 TOK_DATABASELOCATION;
@@ -682,7 +684,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
     xlateMap.put("KW_LIMIT", "LIMIT");
     xlateMap.put("KW_OFFSET", "OFFSET");
     xlateMap.put("KW_SET", "SET");
-    xlateMap.put("KW_PROPERTIES", "TBLPROPERTIES");
+    xlateMap.put("KW_PROPERTIES", "PROPERTIES");
     xlateMap.put("KW_VALUE_TYPE", "\$VALUE\$");
     xlateMap.put("KW_ELEM_TYPE", "\$ELEM\$");
     xlateMap.put("KW_DEFINED", "DEFINED");
@@ -1127,7 +1129,8 @@ createCatalogStatement
         name=identifier
         catLocation
         catalogComment?
-    -> ^(TOK_CREATECATALOG $name catLocation ifNotExists? catalogComment?)
+        (KW_PROPERTIES catprops=catProperties)?
+    -> ^(TOK_CREATECATALOG $name catLocation ifNotExists? catalogComment? $catprops?)
     ;
 
 catLocation
@@ -1144,6 +1147,13 @@ catalogComment
     -> ^(TOK_CATALOGCOMMENT $comment)
     ;
 
+catProperties
+@init { pushMsg("catproperties", state); }
+@after { popMsg(state); }
+    :
+      LPAREN dbPropertiesList RPAREN -> ^(TOK_CATALOGPROPERTIES dbPropertiesList)
+    ;
+
 dropCatalogStatement
 @init { pushMsg("drop catalog statement", state); }
 @after { popMsg(state); }

parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g

@@ -974,7 +974,7 @@ nonReserved
     KW_ABORT | KW_ADD | KW_ADMIN | KW_AFTER | KW_ANALYZE | KW_ARCHIVE | KW_ASC | KW_BEFORE | KW_BUCKET | KW_BUCKETS
     | KW_CASCADE | KW_CBO | KW_CHANGE | KW_CHECK | KW_CLUSTER | KW_CLUSTERED | KW_CLUSTERSTATUS | KW_COLLECTION | KW_COLUMNS
     | KW_COMMENT | KW_COMPACT | KW_COMPACTIONS | KW_COMPUTE | KW_CONCATENATE | KW_CONTINUE | KW_COST | KW_DATA | KW_DAY | KW_CATALOG | KW_CATALOGS
-    | KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DCPROPERTIES | KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY
+    | KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DCPROPERTIES |KW_PROPERTIES | KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY
     | KW_DESC | KW_DIRECTORIES | KW_DIRECTORY | KW_DISABLE | KW_DISTRIBUTE | KW_DISTRIBUTED | KW_DOW | KW_ELEM_TYPE
     | KW_ENABLE | KW_ENFORCED | KW_ESCAPED | KW_EXCLUSIVE | KW_EXPLAIN | KW_EXPORT | KW_FIELDS | KW_FILE | KW_FILEFORMAT
     | KW_FIRST | KW_FORMAT | KW_FORMATTED | KW_FUNCTIONS | KW_HOUR | KW_IDXPROPERTIES | KW_RESPECT | KW_IGNORE

ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/alter/properties/AlterCatalogSetPropertiesAnalyzer.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.catalog.alter.properties;
+
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
+import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogAnalyzer;
+import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogDesc;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import java.util.Map;
+
+/**
+ * Analyzer for catalog set properties commands.
+ */
+@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERCATALOG_PROPERTIES)
+public class AlterCatalogSetPropertiesAnalyzer extends AbstractAlterCatalogAnalyzer {
+    public AlterCatalogSetPropertiesAnalyzer(QueryState queryState) throws SemanticException {
+        super(queryState);
+    }
+
+    @Override
+    protected AbstractAlterCatalogDesc buildAlterCatalogDesc(ASTNode root) throws SemanticException {
+        String catalogName = unescapeIdentifier(root.getChild(0).getText());
+
+        Map<String, String> catProps = null;
+        for (int i = 1; i < root.getChildCount(); i++) {
+            ASTNode childNode = (ASTNode) root.getChild(i);
+            if (childNode.getToken().getType() == HiveParser.TOK_CATALOGPROPERTIES) {
+                catProps = getProps((ASTNode) childNode.getChild(0));
+                break;
+            } else {
+                throw new SemanticException("Unrecognized token in ALTER CATALOG statement");
+            }
+        }
+
+        return new AlterCatalogSetPropertiesDesc(catalogName, catProps);
+    }
+}

ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/alter/properties/AlterCatalogSetPropertiesDesc.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.catalog.alter.properties;
+
+import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogDesc;
+import org.apache.hadoop.hive.ql.plan.Explain;
+
+import java.util.Map;
+
+/**
+ * DDL task description for ALTER CATALOG ... SET PROPERTIES ... commands.
+ */
+@Explain(displayName = "Set Catalog Properties", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED })
+public class AlterCatalogSetPropertiesDesc extends AbstractAlterCatalogDesc {
+    private static final long serialVersionUID = 1L;
+
+    private final Map<String, String> catProperties;
+
+    public AlterCatalogSetPropertiesDesc(String catalogName, Map<String, String> catProperties) {
+        super(catalogName);
+        this.catProperties = catProperties;
+    }
+
+    @Explain(displayName="properties")
+    public Map<String, String> getCatalogProperties() {
+        return catProperties;
+    }
+}

ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/alter/properties/AlterCatalogSetPropertiesOperation.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.catalog.alter.properties;
+
+import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
+import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogOperation;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+
+import java.util.Map;
+
+/**
+ * Analyzer for catalog set properties commands.
+ */
+@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERCATALOG_PROPERTIES)
+public class AlterCatalogSetPropertiesOperation
+        extends AbstractAlterCatalogOperation<AlterCatalogSetPropertiesDesc> {
+    public AlterCatalogSetPropertiesOperation(DDLOperationContext context, AlterCatalogSetPropertiesDesc desc) {
+        super(context, desc);
+    }
+
+    @Override
+    protected void doAlteration(Catalog catalog) throws HiveException {
+        Map<String, String> newParams = desc.getCatalogProperties();
+        Map<String, String> params = catalog.getParameters();
+
+        // if both old and new params are not null, merge them
+        if (params != null && newParams != null) {
+            params.putAll(newParams);
+            catalog.setParameters(params);
+        } else {
+            // if one of them is null, replace the old params with the new one
+            catalog.setParameters(newParams);
+        }
+    }
+}

ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/create/CreateCatalogAnalyzer.java

@@ -29,6 +29,8 @@
 import org.apache.hadoop.hive.ql.parse.HiveParser;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 
+import java.util.Map;
+
 /**
  * Analyzer for catalog creation commands.
  */
@@ -46,6 +48,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
 
     boolean ifNotExists = false;
     String comment = null;
+    Map<String, String> props = null;
 
     for (int i = 2; i < root.getChildCount(); i++) {
       ASTNode childNode = (ASTNode) root.getChild(i);
@@ -56,12 +59,15 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
         case HiveParser.TOK_CATALOGCOMMENT:
           comment = unescapeSQLString(childNode.getChild(0).getText());
           break;
+        case HiveParser.TOK_CATALOGPROPERTIES:
+          props = getProps((ASTNode) childNode.getChild(0));
+          break;
         default:
           throw new SemanticException("Unrecognized token in CREATE CATALOG statement");
       }
     }
 
-    CreateCatalogDesc desc = new CreateCatalogDesc(catalogName, comment, locationUrl, ifNotExists);
+    CreateCatalogDesc desc = new CreateCatalogDesc(catalogName, comment, locationUrl, ifNotExists, props);
     Catalog catalog = new Catalog(catalogName, locationUrl);
 
     rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));

ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/create/CreateCatalogDesc.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.hive.ql.plan.Explain;
 
 import java.io.Serializable;
+import java.util.Map;
 
 /**
  * DDL task description for CREATE CATALOG commands.
@@ -34,12 +35,14 @@ public class CreateCatalogDesc implements DDLDesc, Serializable {
   private final String comment;
   private final String locationUri;
   private final boolean ifNotExists;
+  private final Map<String, String> catProperties;
 
-  public CreateCatalogDesc(String catalogName, String comment, String locationUri, boolean ifNotExists) {
+  public CreateCatalogDesc(String catalogName, String comment, String locationUri, boolean ifNotExists, Map<String, String> dcProperties) {
     this.catalogName = catalogName;
     this.comment = comment;
     this.locationUri = locationUri;
     this.ifNotExists = ifNotExists;
+    this.catProperties = dcProperties;
   }
 
   @Explain(displayName="name", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED })
@@ -61,4 +64,8 @@ public String getLocationUri() {
   public boolean isIfNotExists() {
     return ifNotExists;
   }
+
+  public Map<String, String> getCatlogProperties() {
+    return catProperties;
+  }
 }

ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/create/CreateCatalogOperation.java

@@ -25,6 +25,8 @@
 import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 
+import java.util.Optional;
+
 /**
  * Operation process of creating a catalog.
  */
@@ -37,7 +39,8 @@ public CreateCatalogOperation(DDLOperationContext context, CreateCatalogDesc des
   public int execute() throws Exception {
     Catalog catalog = new Catalog(desc.getName(), desc.getLocationUri());
     catalog.setDescription(desc.getComment());
-
+    Optional.ofNullable(desc.getCatlogProperties())
+            .ifPresent(catalog::setParameters);
     try {
       context.getDb().createCatalog(catalog, desc.isIfNotExists());
     } catch (AlreadyExistsException e) {

ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java

@@ -172,6 +172,7 @@ public enum HiveOperation {
   QUERY("QUERY", HiveParser.TOK_QUERY, new Privilege[]{Privilege.SELECT},
       new Privilege[]{Privilege.ALTER_DATA, Privilege.CREATE}, true, false),
   ALTERCATALOG_LOCATION("ALTERCATALOG_LOCATION", HiveParser.TOK_ALTERCATALOG_LOCATION, new Privilege[]{Privilege.ALTER_METADATA}, null),
+  ALTERCATALOG_PROPERTIES("ALTERCATALOG_PROPERTIES", HiveParser.TOK_ALTERCATALOG_PROPERTIES, null, null),
   ALTERDATABASE("ALTERDATABASE", HiveParser.TOK_ALTERDATABASE_PROPERTIES, null, null),
   ALTERDATABASE_OWNER("ALTERDATABASE_OWNER", HiveParser.TOK_ALTERDATABASE_OWNER, null, null),
   ALTERDATABASE_LOCATION("ALTERDATABASE_LOCATION",

ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java

@@ -126,6 +126,7 @@ public enum HiveOperationType {
   CREATETABLE_AS_SELECT,
   QUERY,
   ALTERCATALOG_LOCATION,
+  ALTERCATALOG_PROPERTIES,
   ALTERDATABASE,
   ALTERDATABASE_OWNER,
   ALTERDATABASE_LOCATION,

ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java

@@ -479,6 +479,7 @@ public HivePrivilegeObjectType getObjectType() {
         new PrivRequirement(ADMIN_PRIV_AR, IOType.OUTPUT)));
     op2Priv.put(HiveOperationType.DROPCATALOG, PrivRequirement.newIOPrivRequirement(null, ADMIN_PRIV_AR));
     op2Priv.put(HiveOperationType.ALTERCATALOG_LOCATION, PrivRequirement.newIOPrivRequirement(null, ADMIN_PRIV_AR));
+    op2Priv.put(HiveOperationType.ALTERCATALOG_PROPERTIES, PrivRequirement.newIOPrivRequirement(null, ADMIN_PRIV_AR));
     op2Priv.put(HiveOperationType.DESCCATALOG, PrivRequirement.newIOPrivRequirement(null, null));
     op2Priv.put(HiveOperationType.SHOWCATALOGS, PrivRequirement.newIOPrivRequirement(null, null));
   }

ql/src/test/queries/clientpositive/catalog.q

@@ -19,7 +19,7 @@ DROP CATALOG test_cat;
 SHOW CATALOGS;
 
 -- CREATE INE doesn't exist
-CREATE CATALOG IF NOT EXISTS test_cat LOCATION '/tmp/test_cat' COMMENT 'Hive test catalog';
+CREATE CATALOG IF NOT EXISTS test_cat LOCATION '/tmp/test_cat' COMMENT 'Hive test catalog' PROPERTIES('key1'='value1');;
 SHOW CATALOGS;
 
 -- DROP IE exists
@@ -45,3 +45,7 @@ SHOW CATALOGS LIKE 'test__';
 -- ALTER LOCATION
 ALTER CATALOG test_cat SET LOCATION '/tmp/test_cat_new';
 DESC CATALOG EXTENDED test_cat;
+
+-- ALTER PROPERTIES.
+-- TODO catalog. Check the catalog's properties after we implement 'desc formatted' or 'show create catalog'.
+ALTER CATALOG test_cat SET PROPERTIES ('key2'='value2');

ql/src/test/results/clientpositive/llap/catalog.q.out

@@ -116,3 +116,9 @@ POSTHOOK: Input: catalog:test_cat
 Catalog Name	test_cat	 	 
 Comment	Hive test catalog	 	 
 #### A masked pattern was here ####
+PREHOOK: query: ALTER CATALOG test_cat SET PROPERTIES ('key2'='value2')
+PREHOOK: type: ALTERCATALOG_PROPERTIES
+PREHOOK: Output: catalog:test_cat
+POSTHOOK: query: ALTER CATALOG test_cat SET PROPERTIES ('key2'='value2')
+POSTHOOK: type: ALTERCATALOG_PROPERTIES
+POSTHOOK: Output: catalog:test_cat