Merge branch 'master' of https://github.com/apache/superset

.github/workflows/bashlib.sh

@@ -212,7 +212,7 @@ cypress-run-applitools() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  $cypress --spec "cypress/e2e/*/**/*.applitools.test.ts" --browser "$browser" --headless
+  $cypress --spec "cypress/applitools/**/*" --browser "$browser" --headless
 
   say "::group::Flask log for default run"
   cat "$flasklog"

UPDATING.md

@@ -84,6 +84,7 @@ assists people when migrating to a new version.
 ### Potential Downtime
 
 - [26416](https://github.com/apache/superset/pull/26416): Adds two database indexes to the `report_execution_log` table and one database index to the `report_recipient` to improve performance. Scheduled downtime may be required for large deployments.
+- [28482](https://github.com/apache/superset/pull/28482): Potentially augments the `query.executed_sql` and `query.select_sql` columns for MySQL from `MEDIUMTEXT` to `LONGTEXT`. Potential downtime may be required for large deployments which previously ran [27119](https://github.com/apache/superset/pull/27119).
 
 ## 3.1.0
 

requirements/base.txt

@@ -52,7 +52,7 @@ cffi==1.15.1
     # via
     #   cryptography
     #   pynacl
-charset-normalizer==3.2.0
+charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via
@@ -185,7 +185,7 @@ korean-lunar-calendar==0.3.1
     # via holidays
 limits==3.4.0
     # via flask-limiter
-llvmlite==0.40.1
+llvmlite==0.42.0
     # via numba
 mako==1.3.3
     # via
@@ -215,7 +215,7 @@ msgspec==0.18.6
     # via flask-session
 nh3==0.2.17
     # via apache-superset
-numba==0.57.1
+numba==0.59.1
     # via pandas
 numexpr==2.10.0
     # via

requirements/development.txt

@@ -48,7 +48,7 @@ db-dtypes==1.1.1
     # via pandas-gbq
 decorator==5.1.1
     # via ipython
-dill==0.3.6
+dill==0.3.8
     # via pylint
 distlib==0.3.6
     # via virtualenv
@@ -131,7 +131,7 @@ jmespath==1.0.1
     #   botocore
 jsonlines==3.1.0
     # via dataflows-tabulator
-jsonschema-spec==0.1.4
+jsonschema-spec==0.1.6
     # via openapi-spec-validator
 kiwisolver==1.4.4
     # via matplotlib
@@ -147,7 +147,7 @@ mccabe==0.7.0
     # via pylint
 mysqlclient==2.2.4
     # via apache-superset
-nodeenv==1.7.0
+nodeenv==1.8.0
     # via pre-commit
 oauthlib==3.2.2
     # via requests-oauthlib

superset-frontend/cypress-base/cypress.config.ts

@@ -65,8 +65,11 @@ export default eyesPlugin(
         return require('./cypress/plugins/index.js')(on, config);
       },
       baseUrl: 'http://localhost:8088',
-      excludeSpecPattern: ['**/*.applitools.test.ts'],
-      specPattern: ['cypress/e2e/**/*.{js,jsx,ts,tsx}'],
+      excludeSpecPattern: [],
+      specPattern: [
+        'cypress/e2e/**/*.{js,jsx,ts,tsx}',
+        'cypress/applitools/**/*.{js,jsx,ts,tsx}',
+      ],
     },
   }),
 );

superset-frontend/cypress-base/cypress/e2e/dashboard/dashboard.applitools.test.ts → superset-frontend/cypress-base/cypress/applitools/dashboard.test.ts

@@ -18,7 +18,7 @@
  */
 import { WORLD_HEALTH_DASHBOARD } from 'cypress/utils/urls';
 import { waitForChartLoad } from 'cypress/utils';
-import { WORLD_HEALTH_CHARTS } from './utils';
+import { WORLD_HEALTH_CHARTS } from '../e2e/dashboard/utils';
 
 describe('Dashboard load', () => {
   beforeEach(() => {

superset-frontend/cypress-base/cypress/e2e/explore/explore.applitools.test.ts → superset-frontend/cypress-base/cypress/applitools/explore.test.ts

@@ -16,7 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { FORM_DATA_DEFAULTS, NUM_METRIC } from './visualizations/shared.helper';
+import {
+  FORM_DATA_DEFAULTS,
+  NUM_METRIC,
+} from '../e2e/explore/visualizations/shared.helper';
 
 describe('explore view', () => {
   beforeEach(() => {

superset/annotation_layers/annotations/schemas.py

@@ -20,7 +20,7 @@
 from marshmallow.validate import Length
 
 from superset.exceptions import SupersetException
-from superset.utils import core as utils
+from superset.utils import json as json_utils
 
 openapi_spec_methods_override = {
     "get": {"get": {"summary": "Get an annotation layer"}},
@@ -51,7 +51,7 @@
 
 def validate_json(value: Union[bytes, bytearray, str]) -> None:
     try:
-        utils.validate_json(value)
+        json_utils.validate_json(value)
     except SupersetException as ex:
         raise ValidationError("JSON not valid") from ex
 

superset/charts/data/api.py

@@ -21,7 +21,6 @@
 import logging
 from typing import Any, TYPE_CHECKING
 
-import simplejson
 from flask import current_app, g, make_response, request, Response
 from flask_appbuilder.api import expose, protect
 from flask_babel import gettext as _
@@ -47,11 +46,11 @@
 from superset.exceptions import QueryObjectValidationError
 from superset.extensions import event_logger
 from superset.models.sql_lab import Query
+from superset.utils import json as json_utils
 from superset.utils.core import (
     create_zip,
     DatasourceType,
     get_user_id,
-    json_int_dttm_ser,
 )
 from superset.utils.decorators import logs_context
 from superset.views.base import CsvResponse, generate_download_headers, XlsxResponse
@@ -396,9 +395,9 @@ def _process_data(query_data: Any) -> Any:
             )
 
         if result_format == ChartDataResultFormat.JSON:
-            response_data = simplejson.dumps(
+            response_data = json_utils.dumps(
                 {"result": result["queries"]},
-                default=json_int_dttm_ser,
+                default=json_utils.json_int_dttm_ser,
                 ignore_nan=True,
             )
             resp = make_response(response_data, 200)

superset/common/query_object.py

@@ -43,10 +43,10 @@
     get_column_names,
     get_metric_names,
     is_adhoc_metric,
-    json_int_dttm_ser,
     QueryObjectFilterClause,
 )
 from superset.utils.hashing import md5_sha_from_dict
+from superset.utils.json import json_int_dttm_ser
 
 if TYPE_CHECKING:
     from superset.connectors.sqla.models import BaseDatasource

superset/dashboards/schemas.py

@@ -24,7 +24,7 @@
 from superset import security_manager
 from superset.exceptions import SupersetException
 from superset.tags.models import TagType
-from superset.utils import core as utils
+from superset.utils import json as json_utils
 
 get_delete_ids_schema = {"type": "array", "items": {"type": "integer"}}
 get_export_ids_schema = {"type": "array", "items": {"type": "integer"}}
@@ -88,7 +88,7 @@
 
 def validate_json(value: Union[bytes, bytearray, str]) -> None:
     try:
-        utils.validate_json(value)
+        json_utils.validate_json(value)
     except SupersetException as ex:
         raise ValidationError("JSON not valid") from ex
 

superset/embedded/view.py

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import json
 from typing import Callable
 
 from flask import abort, request
@@ -25,7 +24,7 @@
 from superset import event_logger, is_feature_enabled
 from superset.daos.dashboard import EmbeddedDashboardDAO
 from superset.superset_typing import FlaskResponse
-from superset.utils import core as utils
+from superset.utils import json as json_utils
 from superset.views.base import BaseSupersetView, common_bootstrap_payload
 
 
@@ -87,7 +86,7 @@ def embedded(
         return self.render_template(
             "superset/spa.html",
             entry="embedded",
-            bootstrap_data=json.dumps(
-                bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser
+            bootstrap_data=json_utils.dumps(
+                bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser
             ),
         )

superset/key_value/utils.py

@@ -26,7 +26,7 @@
 
 from superset.key_value.exceptions import KeyValueParseKeyError
 from superset.key_value.types import KeyValueFilter, KeyValueResource
-from superset.utils.core import json_dumps_w_dates
+from superset.utils.json import json_dumps_w_dates
 
 HASHIDS_MIN_LENGTH = 11
 

superset/migrations/versions/2024-05-09_19-19_f7b6750b67e8_change_mediumtext_to_longtext.py

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""change_mediumtext_to_longtext
+Revision ID: f7b6750b67e8
+Revises: f84fde59123a
+Create Date: 2024-05-09 19:19:46.630140
+"""
+
+# revision identifiers, used by Alembic.
+revision = "f7b6750b67e8"
+down_revision = "f84fde59123a"
+
+from alembic import op  # noqa: E402
+from sqlalchemy.dialects.mysql import MEDIUMTEXT  # noqa: E402
+from sqlalchemy.dialects.mysql.base import MySQLDialect  # noqa: E402
+
+from superset.migrations.shared.utils import get_table_column  # noqa: E402
+from superset.utils.core import LongText, MediumText  # noqa: E402
+
+
+def upgrade():
+    if isinstance(op.get_bind().dialect, MySQLDialect):
+        for item in ["query.executed_sql", "query.select_sql"]:
+            table_name, column_name = item.split(".")
+
+            if (column := get_table_column(table_name, column_name)) and isinstance(
+                column["type"],
+                MEDIUMTEXT,
+            ):
+                with op.batch_alter_table(table_name) as batch_op:
+                    batch_op.alter_column(
+                        column_name,
+                        existing_type=MediumText(),
+                        type_=LongText(),
+                        existing_nullable=True,
+                    )
+
+
+def downgrade():
+    pass

superset/models/core.py

@@ -76,7 +76,7 @@
 from superset.result_set import SupersetResultSet
 from superset.sql_parse import Table
 from superset.superset_typing import OAuth2ClientConfig, ResultSetColumnType
-from superset.utils import cache as cache_util, core as utils
+from superset.utils import cache as cache_util, core as utils, json as json_utils
 from superset.utils.backports import StrEnum
 from superset.utils.core import DatasourceName, get_username
 from superset.utils.oauth2 import get_oauth2_access_token
@@ -601,7 +601,7 @@ def column_needs_conversion(df_series: pd.Series) -> bool:
 
         for col, coltype in df.dtypes.to_dict().items():
             if coltype == numpy.object_ and column_needs_conversion(df[col]):
-                df[col] = df[col].apply(utils.json_dumps_w_dates)
+                df[col] = df[col].apply(json_utils.json_dumps_w_dates)
         return df
 
     @property
@@ -957,7 +957,7 @@ def get_pk_constraint(self, table: Table) -> dict[str, Any]:
 
             def _convert(value: Any) -> Any:
                 try:
-                    return utils.base_json_conv(value)
+                    return json_utils.base_json_conv(value)
                 except TypeError:
                     return None
 

superset/models/dashboard.py

@@ -51,7 +51,7 @@
 from superset.tasks.thumbnails import cache_dashboard_thumbnail
 from superset.tasks.utils import get_current_user
 from superset.thumbnails.digest import get_dashboard_digest
-from superset.utils import core as utils
+from superset.utils import core as utils, json as json_utils
 
 metadata = Model.metadata  # pylint: disable=no-member
 config = app.config
@@ -372,7 +372,7 @@ def export_dashboards(  # pylint: disable=too-many-locals
 
         return json.dumps(
             {"dashboards": copied_dashboards, "datasources": eager_datasources},
-            cls=utils.DashboardEncoder,
+            cls=json_utils.DashboardEncoder,
             indent=4,
         )
 

superset/result_set.py

@@ -28,7 +28,7 @@
 
 from superset.db_engine_specs import BaseEngineSpec
 from superset.superset_typing import DbapiDescription, DbapiResult, ResultSetColumnType
-from superset.utils import core as utils
+from superset.utils import core as utils, json as json_utils
 from superset.utils.core import GenericDataType
 
 logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@ def dedup(l: list[str], suffix: str = "__", case_sensitive: bool = True) -> list
 
 
 def stringify(obj: Any) -> str:
-    return json.dumps(obj, default=utils.json_iso_dttm_ser)
+    return json_utils.dumps(obj, default=json_utils.json_iso_dttm_ser)
 
 
 def stringify_values(array: NDArray[Any]) -> NDArray[Any]:

superset/sql_lab.py

@@ -24,7 +24,6 @@
 
 import backoff
 import msgpack
-import simplejson as json
 from celery.exceptions import SoftTimeLimitExceeded
 from flask_babel import gettext as __
 
@@ -59,8 +58,8 @@
 )
 from superset.sqllab.limiting_factor import LimitingFactor
 from superset.sqllab.utils import write_ipc_buffer
+from superset.utils import json as json_utils
 from superset.utils.core import (
-    json_iso_dttm_ser,
     override_user,
     QuerySource,
     zlib_compress,
@@ -349,9 +348,13 @@ def _serialize_payload(
 ) -> Union[bytes, str]:
     logger.debug("Serializing to msgpack: %r", use_msgpack)
     if use_msgpack:
-        return msgpack.dumps(payload, default=json_iso_dttm_ser, use_bin_type=True)
+        return msgpack.dumps(
+            payload, default=json_utils.json_iso_dttm_ser, use_bin_type=True
+        )
 
-    return json.dumps(payload, default=json_iso_dttm_ser, ignore_nan=True)
+    return json_utils.dumps(
+        payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True
+    )
 
 
 def _serialize_and_expand_data(