Skip to content

Commit 5ecd669

Browse files
committed
Merge branch 'master' of https://github.com/apache/superset
2 parents 955034e + 1c48fe0 commit 5ecd669

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+503
-327
lines changed

.github/workflows/bashlib.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ cypress-run-applitools() {
212212
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
213213
local flaskProcessId=$!
214214

215-
$cypress --spec "cypress/e2e/*/**/*.applitools.test.ts" --browser "$browser" --headless
215+
$cypress --spec "cypress/applitools/**/*" --browser "$browser" --headless
216216

217217
say "::group::Flask log for default run"
218218
cat "$flasklog"

UPDATING.md

+1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ assists people when migrating to a new version.
8484
### Potential Downtime
8585

8686
- [26416](https://github.com/apache/superset/pull/26416): Adds two database indexes to the `report_execution_log` table and one database index to the `report_recipient` to improve performance. Scheduled downtime may be required for large deployments.
87+
- [28482](https://github.com/apache/superset/pull/28482): Potentially augments the `query.executed_sql` and `query.select_sql` columns for MySQL from `MEDIUMTEXT` to `LONGTEXT`. Potential downtime may be required for large deployments which previously ran [27119](https://github.com/apache/superset/pull/27119).
8788

8889
## 3.1.0
8990

requirements/base.txt

+3-3
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ cffi==1.15.1
5252
# via
5353
# cryptography
5454
# pynacl
55-
charset-normalizer==3.2.0
55+
charset-normalizer==3.3.2
5656
# via requests
5757
click==8.1.7
5858
# via
@@ -185,7 +185,7 @@ korean-lunar-calendar==0.3.1
185185
# via holidays
186186
limits==3.4.0
187187
# via flask-limiter
188-
llvmlite==0.40.1
188+
llvmlite==0.42.0
189189
# via numba
190190
mako==1.3.3
191191
# via
@@ -215,7 +215,7 @@ msgspec==0.18.6
215215
# via flask-session
216216
nh3==0.2.17
217217
# via apache-superset
218-
numba==0.57.1
218+
numba==0.59.1
219219
# via pandas
220220
numexpr==2.10.0
221221
# via

requirements/development.txt

+3-3
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ db-dtypes==1.1.1
4848
# via pandas-gbq
4949
decorator==5.1.1
5050
# via ipython
51-
dill==0.3.6
51+
dill==0.3.8
5252
# via pylint
5353
distlib==0.3.6
5454
# via virtualenv
@@ -131,7 +131,7 @@ jmespath==1.0.1
131131
# botocore
132132
jsonlines==3.1.0
133133
# via dataflows-tabulator
134-
jsonschema-spec==0.1.4
134+
jsonschema-spec==0.1.6
135135
# via openapi-spec-validator
136136
kiwisolver==1.4.4
137137
# via matplotlib
@@ -147,7 +147,7 @@ mccabe==0.7.0
147147
# via pylint
148148
mysqlclient==2.2.4
149149
# via apache-superset
150-
nodeenv==1.7.0
150+
nodeenv==1.8.0
151151
# via pre-commit
152152
oauthlib==3.2.2
153153
# via requests-oauthlib

superset-frontend/cypress-base/cypress.config.ts

+5-2
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,11 @@ export default eyesPlugin(
6565
return require('./cypress/plugins/index.js')(on, config);
6666
},
6767
baseUrl: 'http://localhost:8088',
68-
excludeSpecPattern: ['**/*.applitools.test.ts'],
69-
specPattern: ['cypress/e2e/**/*.{js,jsx,ts,tsx}'],
68+
excludeSpecPattern: [],
69+
specPattern: [
70+
'cypress/e2e/**/*.{js,jsx,ts,tsx}',
71+
'cypress/applitools/**/*.{js,jsx,ts,tsx}',
72+
],
7073
},
7174
}),
7275
);

superset-frontend/cypress-base/cypress/e2e/dashboard/dashboard.applitools.test.ts superset-frontend/cypress-base/cypress/applitools/dashboard.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919
import { WORLD_HEALTH_DASHBOARD } from 'cypress/utils/urls';
2020
import { waitForChartLoad } from 'cypress/utils';
21-
import { WORLD_HEALTH_CHARTS } from './utils';
21+
import { WORLD_HEALTH_CHARTS } from '../e2e/dashboard/utils';
2222

2323
describe('Dashboard load', () => {
2424
beforeEach(() => {

superset-frontend/cypress-base/cypress/e2e/explore/explore.applitools.test.ts superset-frontend/cypress-base/cypress/applitools/explore.test.ts

+4-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
import { FORM_DATA_DEFAULTS, NUM_METRIC } from './visualizations/shared.helper';
19+
import {
20+
FORM_DATA_DEFAULTS,
21+
NUM_METRIC,
22+
} from '../e2e/explore/visualizations/shared.helper';
2023

2124
describe('explore view', () => {
2225
beforeEach(() => {

superset/annotation_layers/annotations/schemas.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from marshmallow.validate import Length
2121

2222
from superset.exceptions import SupersetException
23-
from superset.utils import core as utils
23+
from superset.utils import json as json_utils
2424

2525
openapi_spec_methods_override = {
2626
"get": {"get": {"summary": "Get an annotation layer"}},
@@ -51,7 +51,7 @@
5151

5252
def validate_json(value: Union[bytes, bytearray, str]) -> None:
5353
try:
54-
utils.validate_json(value)
54+
json_utils.validate_json(value)
5555
except SupersetException as ex:
5656
raise ValidationError("JSON not valid") from ex
5757

superset/charts/data/api.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import logging
2222
from typing import Any, TYPE_CHECKING
2323

24-
import simplejson
2524
from flask import current_app, g, make_response, request, Response
2625
from flask_appbuilder.api import expose, protect
2726
from flask_babel import gettext as _
@@ -47,11 +46,11 @@
4746
from superset.exceptions import QueryObjectValidationError
4847
from superset.extensions import event_logger
4948
from superset.models.sql_lab import Query
49+
from superset.utils import json as json_utils
5050
from superset.utils.core import (
5151
create_zip,
5252
DatasourceType,
5353
get_user_id,
54-
json_int_dttm_ser,
5554
)
5655
from superset.utils.decorators import logs_context
5756
from superset.views.base import CsvResponse, generate_download_headers, XlsxResponse
@@ -396,9 +395,9 @@ def _process_data(query_data: Any) -> Any:
396395
)
397396

398397
if result_format == ChartDataResultFormat.JSON:
399-
response_data = simplejson.dumps(
398+
response_data = json_utils.dumps(
400399
{"result": result["queries"]},
401-
default=json_int_dttm_ser,
400+
default=json_utils.json_int_dttm_ser,
402401
ignore_nan=True,
403402
)
404403
resp = make_response(response_data, 200)

superset/common/query_object.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@
4343
get_column_names,
4444
get_metric_names,
4545
is_adhoc_metric,
46-
json_int_dttm_ser,
4746
QueryObjectFilterClause,
4847
)
4948
from superset.utils.hashing import md5_sha_from_dict
49+
from superset.utils.json import json_int_dttm_ser
5050

5151
if TYPE_CHECKING:
5252
from superset.connectors.sqla.models import BaseDatasource

superset/dashboards/schemas.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from superset import security_manager
2525
from superset.exceptions import SupersetException
2626
from superset.tags.models import TagType
27-
from superset.utils import core as utils
27+
from superset.utils import json as json_utils
2828

2929
get_delete_ids_schema = {"type": "array", "items": {"type": "integer"}}
3030
get_export_ids_schema = {"type": "array", "items": {"type": "integer"}}
@@ -88,7 +88,7 @@
8888

8989
def validate_json(value: Union[bytes, bytearray, str]) -> None:
9090
try:
91-
utils.validate_json(value)
91+
json_utils.validate_json(value)
9292
except SupersetException as ex:
9393
raise ValidationError("JSON not valid") from ex
9494

superset/embedded/view.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
import json
1817
from typing import Callable
1918

2019
from flask import abort, request
@@ -25,7 +24,7 @@
2524
from superset import event_logger, is_feature_enabled
2625
from superset.daos.dashboard import EmbeddedDashboardDAO
2726
from superset.superset_typing import FlaskResponse
28-
from superset.utils import core as utils
27+
from superset.utils import json as json_utils
2928
from superset.views.base import BaseSupersetView, common_bootstrap_payload
3029

3130

@@ -87,7 +86,7 @@ def embedded(
8786
return self.render_template(
8887
"superset/spa.html",
8988
entry="embedded",
90-
bootstrap_data=json.dumps(
91-
bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser
89+
bootstrap_data=json_utils.dumps(
90+
bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser
9291
),
9392
)

superset/key_value/utils.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626

2727
from superset.key_value.exceptions import KeyValueParseKeyError
2828
from superset.key_value.types import KeyValueFilter, KeyValueResource
29-
from superset.utils.core import json_dumps_w_dates
29+
from superset.utils.json import json_dumps_w_dates
3030

3131
HASHIDS_MIN_LENGTH = 11
3232

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
"""change_mediumtext_to_longtext
18+
Revision ID: f7b6750b67e8
19+
Revises: f84fde59123a
20+
Create Date: 2024-05-09 19:19:46.630140
21+
"""
22+
23+
# revision identifiers, used by Alembic.
24+
revision = "f7b6750b67e8"
25+
down_revision = "f84fde59123a"
26+
27+
from alembic import op # noqa: E402
28+
from sqlalchemy.dialects.mysql import MEDIUMTEXT # noqa: E402
29+
from sqlalchemy.dialects.mysql.base import MySQLDialect # noqa: E402
30+
31+
from superset.migrations.shared.utils import get_table_column # noqa: E402
32+
from superset.utils.core import LongText, MediumText # noqa: E402
33+
34+
35+
def upgrade():
36+
if isinstance(op.get_bind().dialect, MySQLDialect):
37+
for item in ["query.executed_sql", "query.select_sql"]:
38+
table_name, column_name = item.split(".")
39+
40+
if (column := get_table_column(table_name, column_name)) and isinstance(
41+
column["type"],
42+
MEDIUMTEXT,
43+
):
44+
with op.batch_alter_table(table_name) as batch_op:
45+
batch_op.alter_column(
46+
column_name,
47+
existing_type=MediumText(),
48+
type_=LongText(),
49+
existing_nullable=True,
50+
)
51+
52+
53+
def downgrade():
54+
pass

superset/models/core.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@
7676
from superset.result_set import SupersetResultSet
7777
from superset.sql_parse import Table
7878
from superset.superset_typing import OAuth2ClientConfig, ResultSetColumnType
79-
from superset.utils import cache as cache_util, core as utils
79+
from superset.utils import cache as cache_util, core as utils, json as json_utils
8080
from superset.utils.backports import StrEnum
8181
from superset.utils.core import DatasourceName, get_username
8282
from superset.utils.oauth2 import get_oauth2_access_token
@@ -601,7 +601,7 @@ def column_needs_conversion(df_series: pd.Series) -> bool:
601601

602602
for col, coltype in df.dtypes.to_dict().items():
603603
if coltype == numpy.object_ and column_needs_conversion(df[col]):
604-
df[col] = df[col].apply(utils.json_dumps_w_dates)
604+
df[col] = df[col].apply(json_utils.json_dumps_w_dates)
605605
return df
606606

607607
@property
@@ -957,7 +957,7 @@ def get_pk_constraint(self, table: Table) -> dict[str, Any]:
957957

958958
def _convert(value: Any) -> Any:
959959
try:
960-
return utils.base_json_conv(value)
960+
return json_utils.base_json_conv(value)
961961
except TypeError:
962962
return None
963963

superset/models/dashboard.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
from superset.tasks.thumbnails import cache_dashboard_thumbnail
5252
from superset.tasks.utils import get_current_user
5353
from superset.thumbnails.digest import get_dashboard_digest
54-
from superset.utils import core as utils
54+
from superset.utils import core as utils, json as json_utils
5555

5656
metadata = Model.metadata # pylint: disable=no-member
5757
config = app.config
@@ -372,7 +372,7 @@ def export_dashboards( # pylint: disable=too-many-locals
372372

373373
return json.dumps(
374374
{"dashboards": copied_dashboards, "datasources": eager_datasources},
375-
cls=utils.DashboardEncoder,
375+
cls=json_utils.DashboardEncoder,
376376
indent=4,
377377
)
378378

superset/result_set.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
from superset.db_engine_specs import BaseEngineSpec
3030
from superset.superset_typing import DbapiDescription, DbapiResult, ResultSetColumnType
31-
from superset.utils import core as utils
31+
from superset.utils import core as utils, json as json_utils
3232
from superset.utils.core import GenericDataType
3333

3434
logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@ def dedup(l: list[str], suffix: str = "__", case_sensitive: bool = True) -> list
6161

6262

6363
def stringify(obj: Any) -> str:
64-
return json.dumps(obj, default=utils.json_iso_dttm_ser)
64+
return json_utils.dumps(obj, default=json_utils.json_iso_dttm_ser)
6565

6666

6767
def stringify_values(array: NDArray[Any]) -> NDArray[Any]:

superset/sql_lab.py

+7-4
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424

2525
import backoff
2626
import msgpack
27-
import simplejson as json
2827
from celery.exceptions import SoftTimeLimitExceeded
2928
from flask_babel import gettext as __
3029

@@ -59,8 +58,8 @@
5958
)
6059
from superset.sqllab.limiting_factor import LimitingFactor
6160
from superset.sqllab.utils import write_ipc_buffer
61+
from superset.utils import json as json_utils
6262
from superset.utils.core import (
63-
json_iso_dttm_ser,
6463
override_user,
6564
QuerySource,
6665
zlib_compress,
@@ -349,9 +348,13 @@ def _serialize_payload(
349348
) -> Union[bytes, str]:
350349
logger.debug("Serializing to msgpack: %r", use_msgpack)
351350
if use_msgpack:
352-
return msgpack.dumps(payload, default=json_iso_dttm_ser, use_bin_type=True)
351+
return msgpack.dumps(
352+
payload, default=json_utils.json_iso_dttm_ser, use_bin_type=True
353+
)
353354

354-
return json.dumps(payload, default=json_iso_dttm_ser, ignore_nan=True)
355+
return json_utils.dumps(
356+
payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True
357+
)
355358

356359

357360
def _serialize_and_expand_data(

0 commit comments

Comments
 (0)