diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index b1af7344d9320..ab4eb9e6edbc0 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -28,10 +28,22 @@ import materialize.parallel_workload.column from materialize.data_ingest.data_type import ( DATA_TYPES, + DATA_TYPES_FOR_COLUMNS, NUMBER_TYPES, + RANGE_TYPES, + UUID, Boolean, + Bytea, + Char, + IntArray, + IntList, + Jsonb, + Oid, Text, TextTextMap, + Timestamp, + TimestampTz, + VarChar, ) from materialize.data_ingest.query_error import QueryError from materialize.data_ingest.row import Operation @@ -43,6 +55,8 @@ Materialized, ) from materialize.mzcompose.services.minio import minio_blob_uri +from materialize.mzcompose.services.mysql import MySql +from materialize.mzcompose.services.sql_server import SqlServer from materialize.parallel_workload.database import ( DB, MAX_CLUSTER_REPLICAS, @@ -53,13 +67,16 @@ MAX_INDEXES, MAX_KAFKA_SINKS, MAX_KAFKA_SOURCES, + MAX_LOADGEN_SOURCES, MAX_MYSQL_SOURCES, + MAX_NETWORK_POLICIES, MAX_POSTGRES_SOURCES, MAX_ROLES, MAX_ROWS, MAX_SCHEMAS, MAX_SQL_SERVER_SOURCES, MAX_TABLES, + MAX_TYPES, MAX_VIEWS, MAX_WEBHOOK_SOURCES, Cluster, @@ -71,14 +88,18 @@ Index, KafkaSink, KafkaSource, + LoadGeneratorSource, + MultiLoadGeneratorSource, MySqlSource, MzTempSchema, + NetworkPolicy, PostgresSource, Role, S3Object, Schema, SqlServerSource, Table, + Type, View, WebhookSource, ) @@ -139,6 +160,67 @@ def ws_connect(ws: websocket.WebSocket, host, port, user: str) -> tuple[int, int return (ws_conn_id, ws_secret_key) +def untrack_objects_in_schemas(exe: Executor, schemas: set[Schema]) -> None: + """Remove every tracked object living in one of the given schemas. + + Called after a CASCADE drop of a schema or database. Source executor + connections are closed. Cross-schema dependents are cascade-dropped + server-side but not pruned here, they later surface as "does not exist", + which is why the CASCADE actions only run in DDL complexity.""" + with exe.db.lock: + exe.db.tables[:] = [t for t in exe.db.tables if t.schema not in schemas] + exe.db.views[:] = [v for v in exe.db.views if v.schema not in schemas] + # NOTE: Removed in place, like the lists above. DropIndexAction + # discards from this set holding only the index's own lock, so + # rebinding the attribute would resurrect an index that was dropped + # between building the replacement set and assigning it. + exe.db.indexes.difference_update( + {i for i in exe.db.indexes if i.schema in schemas} + ) + # Close the executor connections of the ingestion sources being + # untracked so they don't leak. + connected_sources = ( + exe.db.kafka_sources + + exe.db.postgres_sources + + exe.db.mysql_sources + + exe.db.sql_server_sources + ) + for src in connected_sources: + if src.schema in schemas: + try: + src.executor.mz_conn.close() + except Exception: + pass + exe.db.kafka_sources[:] = [ + s for s in exe.db.kafka_sources if s.schema not in schemas + ] + exe.db.postgres_sources[:] = [ + s for s in exe.db.postgres_sources if s.schema not in schemas + ] + exe.db.mysql_sources[:] = [ + s for s in exe.db.mysql_sources if s.schema not in schemas + ] + exe.db.sql_server_sources[:] = [ + s for s in exe.db.sql_server_sources if s.schema not in schemas + ] + exe.db.loadgen_sources[:] = [ + s for s in exe.db.loadgen_sources if s.schema not in schemas + ] + exe.db.multi_loadgen_sources[:] = [ + s for s in exe.db.multi_loadgen_sources if s.schema not in schemas + ] + exe.db.webhook_sources[:] = [ + s for s in exe.db.webhook_sources if s.schema not in schemas + ] + exe.db.kafka_sinks[:] = [ + s for s in exe.db.kafka_sinks if s.schema not in schemas + ] + exe.db.iceberg_sinks[:] = [ + s for s in exe.db.iceberg_sinks if s.schema not in schemas + ] + exe.db.types[:] = [t for t in exe.db.types if t.schema not in schemas] + + # TODO: CASCADE in DROPs, keep track of what will be deleted class Action: rng: random.Random @@ -201,7 +283,14 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: # catalog apply, which surfaces as an unexpected failure. "non-temporary items cannot depend on temporary item", "is not of expected type SqlColumnType", # TODO: Remove when SQL-566 is fixed + # generate_select_query occasionally emits a WITH MUTUALLY + # RECURSIVE body with ERROR AT RECURSION LIMIT, which errors on + # purpose when the iteration outruns the limit. Only our generated + # queries produce this, so ignoring it globally is safe. + "exceeded the recursion limit", ] + if exe.statement_timeout_set: + result.append("canceling statement due to statement timeout") if exe.db.complexity in (Complexity.DDL, Complexity.DDLOnly): result.extend( [ @@ -214,6 +303,12 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: # matching "unknown cluster replica size" errors. "unknown cluster '", "unknown schema", # schema was dropped + # The database was dropped concurrently: DropDatabaseAction + # can land on an emptied database, and + # DropDatabaseCascadeAction (disabled until SQL-518 is + # fixed) drops non-empty ones. + "unknown database", + "invalid database", # CREATE SCHEMA wording for a vanished database "the transaction's active cluster has been dropped", # cluster was dropped "was removed", # dependency was removed, started with moving optimization off main thread, see database-issues#7285 "real-time source dropped before ingesting the upstream system's visible frontier", # Expected, see https://buildkite.com/materialize/nightly/builds/9399#0191be17-1f4c-4321-9b51-edc4b08b71c5 @@ -285,21 +380,83 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: ) if exe.db.scenario == Scenario.Rename: result.extend(["unknown schema", "ambiguous reference to schema name"]) + # RepeatRow is the only scenario that deliberately produces negative + # multiplicities (`repeat_row` with a negative count), and the + # corruption is observed by any later reader of the affected + # collection, not just by the statement that created it. So the whole + # class is tolerated for the entire scenario, not per action. Every + # other scenario stays strict, a negative-accumulation error there is a + # genuine finding. The central list lives in + # `negative_accumulation_errors.py`. if exe.db.scenario == Scenario.RepeatRow: - # Views that use `repeat_row` with a negative count can surface - # negative-accumulation errors both at DDL time (constant folding) - # and at read time (various operators checking multiplicities). - # The central list lives in `negative_accumulation_errors.py` so - # we can update it in one place when the error messages change. result.extend(NEGATIVE_ACCUMULATION_ERRORS) if materialize.parallel_workload.column.NAUGHTY_IDENTIFIERS: result.extend(["identifier length exceeds 255 bytes"]) return result + # MIN/MAX exist for every type except these (bytea, jsonb, map, list, + # array, uuid, oid, and the range types). + _MINMAX_EXCLUDED = ( + Bytea, + Jsonb, + TextTextMap, + IntList, + IntArray, + UUID, + Oid, + ) + tuple(RANGE_TYPES) + + def aggregate_fns(self, column: Column) -> list[str]: + """Aggregate function templates valid for the column's type. + + Used both in window position (OVER ..) and in GROUP BY position. The + collection aggregates (array_agg/list_agg/jsonb_agg/string_agg) + exercise the "collection" reduce rendering, distinct from the + accumulable sum/count path. Type exclusions are empirically derived, + e.g. array_agg rejects char and cannot nest map/list/array.""" + dt = column.data_type + fns = ["COUNT({})"] + if dt in NUMBER_TYPES: + fns.extend( + [ + "SUM({})", + "AVG({})", + "STDDEV({})", + "STDDEV_POP({})", + "STDDEV_SAMP({})", + "VAR_SAMP({})", + "VAR_POP({})", + ] + ) + if dt == Boolean: + fns.extend(["BOOL_AND({})", "BOOL_OR({})"]) + if dt not in self._MINMAX_EXCLUDED: + fns.extend(["MAX({})", "MIN({})"]) + # Collection aggregates. + fns.append("jsonb_agg({})") + if dt != Char: + fns.append("list_agg({})") + if dt not in (Char, TextTextMap, IntList, IntArray): + fns.append("array_agg({})") + if dt in (Text, VarChar, Char): + fns.append("string_agg({}, ',')") + return fns + def generate_select_query(self, exe: Executor, expr_kind: ExprKind) -> str: - obj = self.rng.choice(exe.db.db_objects()) + objects = exe.db.db_objects() + if not objects: + # A concurrent CASCADE drop removed every object. Nothing to query. + return "SELECT 1" + obj = self.rng.choice(objects) column = self.rng.choice(obj.columns) - obj2 = self.rng.choice(exe.db.db_objects_without_views()) + # db_objects_without_views() can momentarily be empty if a CASCADE drop + # removed the last table/source/MV, leaving only plain views. Fall back + # to obj (only used for an optional join, which is skipped for views). + # The list is taken once and reused below: recomputing it can return an + # empty list even though it was non-empty here, and rng.choice on that + # raises IndexError, which fails the whole run. + objs_without_views = exe.db.db_objects_without_views() + obj2 = self.rng.choice(objs_without_views) if objs_without_views else obj obj_name = str(obj) obj2_name = str(obj2) columns = [ @@ -312,105 +469,219 @@ def generate_select_query(self, exe: Executor, expr_kind: ExprKind) -> str: all_columns = list(obj.columns) + list(obj2.columns) if join else obj.columns - column_types = [] - if self.rng.random() < 0.9: - column_types = [ - self.rng.choice(list(DATA_TYPES)) - for i in range(self.rng.randint(1, 10)) + # Self-contained iterative dataflow, cross joined with a real object. + # The iteration bound and the recursion limit are drawn independently, + # so the limit sometimes fires, exercising the RETURN/ERROR AT paths + # on purpose ("exceeded the recursion limit" is an expected error). + if self.rng.random() < 0.05: + limit_kind = self.rng.choice(["RETURN AT", "ERROR AT"]) + expr = expression( + self.rng.choice(list(DATA_TYPES)), obj.columns, self.rng, expr_kind + ) + return ( + f"WITH MUTUALLY RECURSIVE ({limit_kind} RECURSION LIMIT {self.rng.randint(2, 200)}) " + f"cnt (i int8) AS (" + f"SELECT 1 UNION ALL SELECT i + 1 FROM cnt WHERE i < {self.rng.randint(1, 100)}" + f") SELECT i, {expr} FROM cnt, {obj_name}" + f" LIMIT {self.rng.randint(0, 100)}" + ) + + # Explicit LATERAL derived table correlated on a matching-type column. + # Exercises correlated-subquery decorrelation. + if objs_without_views and self.rng.random() < 0.08: + outer_col = self.rng.choice(obj.columns) + inner_obj = self.rng.choice(objs_without_views) + match_cols = [ + c + for c in inner_obj.columns + if c.data_type == outer_col.data_type and c.data_type != TextTextMap ] - expressions = ", ".join( - [ - expression( - column_type, - all_columns, - self.rng, - expr_kind, - ) - for column_type in column_types - ] + corr = "" + if match_cols: + corr = f" WHERE {self.rng.choice(match_cols)} = {outer_col}" + expr = expression( + self.rng.choice(list(DATA_TYPES)), obj.columns, self.rng, expr_kind + ) + return ( + f"SELECT {expr} FROM {obj_name}, LATERAL (" + f"SELECT count(*) AS cnt FROM {inner_obj}{corr}" + f") AS lat LIMIT {self.rng.randint(0, 100)}" + ) + + # Table function in FROM. unnest of a column reference is an + # implicitly lateral call. + if self.rng.random() < 0.1: + int_list_columns = [c for c in obj.columns if c.data_type == IntList] + if int_list_columns and self.rng.choice([True, False]): + func = f"unnest({self.rng.choice(int_list_columns)})" + else: + func = f"generate_series(1, {self.rng.randint(1, 100)})" + expr = expression( + self.rng.choice(list(DATA_TYPES)), obj.columns, self.rng, expr_kind ) if self.rng.choice([True, False]): - column_types = [] - column1 = self.rng.choice(all_columns) - column2 = self.rng.choice(all_columns) - column3 = self.rng.choice(all_columns) - fns = [ - "COUNT({})", - # "LIST_AGG({})", - # "JSONB_AGG({})", - ] - # if column1.data_type == Text: - # fns.extend(["STRING_AGG({}, ',')"]) - # if column1.data_type not in [TextTextMap, IntArray, IntList]: - # fns.extend(["ARRAY_AGG({})"]) - if column1.data_type in NUMBER_TYPES: - fns.extend( - [ - "SUM({})", - "AVG({})", - "MAX({})", - "MIN({})", - "STDDEV({})", - "STDDEV_POP({})", - "STDDEV_SAMP({})", - "VAR_SAMP({})", - "VAR_POP({})", - ] - ) - elif column1.data_type == Boolean: - fns.extend(["BOOL_AND({})", "BOOL_OR({})"]) - window_fn = self.rng.choice(fns) - expressions += f", {window_fn.format(column1)} OVER (PARTITION BY {column2} ORDER BY {column3})" - else: - expressions = "*" + tf = f"{func} WITH ORDINALITY AS tf(x, ord)" + else: + tf = f"{func} AS tf(x)" + return ( + f"SELECT tf.x, {expr} FROM {obj_name}, {tf}" + f" LIMIT {self.rng.randint(0, 100)}" + ) - query = f"SELECT {expressions} FROM {obj_name}" + star = self.rng.random() >= 0.9 + exprs: list[tuple[type, str]] = [] + if not star: + for i in range(self.rng.randint(1, 10)): + dt = self.rng.choice(list(DATA_TYPES)) + exprs.append((dt, expression(dt, all_columns, self.rng, expr_kind))) if join: + join_kind = self.rng.choice( + ["JOIN", "JOIN", "JOIN", "LEFT JOIN", "RIGHT JOIN", "FULL JOIN"] + ) column2 = self.rng.choice(columns) - query += f" JOIN {obj2_name} ON {column} = {column2}" + join_clause = f" {join_kind} {obj2_name} ON {column} = {column2}" + if self.rng.random() < 0.2: + join_clause += ( + f" AND {expression(Boolean, all_columns, self.rng, expr_kind)}" + ) + else: + join_clause = "" - if self.rng.choice([True, False]): - query += f" WHERE {expression(Boolean, all_columns, self.rng, expr_kind)}" - - if bool(column_types) and self.rng.choice([True, False]): - obj3 = self.rng.choice(exe.db.db_objects()) - obj3_name = str(obj3) - column3 = self.rng.choice(obj3.columns) - obj4 = self.rng.choice(exe.db.db_objects()) - obj4_name = str(obj4) - columns_union = [ - c - for c in obj4.columns - if c.data_type == column3.data_type and c.data_type != TextTextMap + def where_clause() -> str: + parts = [] + if self.rng.choice([True, False]): + parts.append(expression(Boolean, all_columns, self.rng, expr_kind)) + if objs_without_views and self.rng.random() < 0.2: + obj3 = self.rng.choice(objs_without_views) + sub_columns = [ + c + for c in obj3.columns + if c.data_type == column.data_type and c.data_type != TextTextMap + ] + sub_kind = self.rng.choice(["exists", "not exists", "in", "scalar"]) + if sub_kind in ("exists", "not exists"): + cond = expression(Boolean, obj3.columns, self.rng, expr_kind) + parts.append( + f"{'NOT ' if sub_kind == 'not exists' else ''}EXISTS (SELECT 1 FROM {obj3} WHERE {cond})" + ) + elif sub_columns: + sub_column = self.rng.choice(sub_columns) + if sub_kind == "in": + parts.append(f"{column} IN (SELECT {sub_column} FROM {obj3})") + else: + parts.append( + f"{column} = (SELECT {sub_column} FROM {obj3} LIMIT 1)" + ) + # Deliberate temporal filter, mz_now() has to be a top-level + # conjunct for the temporal filter machinery to apply. + ts_columns = [ + c for c in all_columns if c.data_type in (Timestamp, TimestampTz) ] - join_union = ( - obj3_name != obj4_name and obj3 not in exe.db.views and columns_union + if ts_columns and expr_kind != ExprKind.WRITE and self.rng.random() < 0.2: + ts_column = self.rng.choice(ts_columns) + op = self.rng.choice(["<=", ">="]) + parts.append( + f"mz_now() {op} {ts_column} + INTERVAL '{self.rng.randint(0, 100000)} seconds'" + ) + if not parts: + return "" + # Generated Boolean expressions can contain OR, so parenthesize + # each part to keep every part as a top-level conjunct. + return " WHERE " + " AND ".join(f"({part})" for part in parts) + + distinct_on_expr = None + group_by = bool(exprs) and self.rng.random() < 0.2 + if group_by: + num_group = self.rng.randint(1, len(exprs)) + select_list = [expr for _, expr in exprs[:num_group]] + # Only the group keys come from exprs. The aggregates that follow + # get None, their result type is not exprs[i]'s. + select_types: list[type | None] = [dt for dt, _ in exprs[:num_group]] + for i in range(self.rng.randint(1, 3)): + agg_column = self.rng.choice(all_columns) + fn = self.rng.choice(self.aggregate_fns(agg_column)) + select_list.append(fn.format(agg_column)) + select_types.append(None) + group_clause = ( + f" GROUP BY {', '.join(str(i + 1) for i in range(num_group))}" ) - all_columns_union = ( - list(obj3.columns) + list(obj4.columns) if join_union else obj3.columns + # Clause order is fixed: HAVING before OPTIONS. + if self.rng.random() < 0.3: + group_clause += f" HAVING count(*) >= {self.rng.randint(0, 10)}" + if self.rng.random() < 0.3: + group_clause += f" OPTIONS (AGGREGATE INPUT GROUP SIZE = {self.rng.randint(1, 1000)})" + distinct_clause = "" + else: + select_list = [expr for _, expr in exprs] if exprs else ["*"] + # `*` expands to every column, so the output types are the columns'. + select_types = ( + [dt for dt, _ in exprs] if exprs else [c.data_type for c in all_columns] ) - expressions3 = ", ".join( - [ - expression( - column_type, - all_columns_union, - self.rng, - expr_kind, - ) - for column_type in column_types - ] + group_clause = "" + distinct_clause = "" + distinct_on_expr = None + if exprs and self.rng.random() < 0.15: + if exprs[0][0] == TextTextMap or self.rng.choice([True, False]): + distinct_clause = "DISTINCT " + else: + # The DISTINCT ON expressions must be a prefix of the + # ORDER BY expressions. The leading ORDER BY item repeats + # this expression verbatim (not its ordinal) so the match + # holds even for bare literals. + distinct_on_expr = exprs[0][1] + distinct_clause = f"DISTINCT ON ({distinct_on_expr}) " + if exprs and self.rng.choice([True, False]): + column1 = self.rng.choice(all_columns) + column2 = self.rng.choice(all_columns) + column3 = self.rng.choice(all_columns) + window_fn = self.rng.choice(self.aggregate_fns(column1)) + select_list.append( + f"{window_fn.format(column1)} OVER (PARTITION BY {column2} ORDER BY {column3})" + ) + select_types.append(None) + + expressions = ", ".join(select_list) + query = f"SELECT {distinct_clause}{expressions} FROM {obj_name}" + query += join_clause + query += where_clause() + query += group_clause + + if not distinct_clause and self.rng.choice([True, False]): + set_op = self.rng.choice( + ["UNION ALL"] * 4 + + ["UNION", "INTERSECT", "INTERSECT ALL", "EXCEPT", "EXCEPT ALL"] ) - query += f" UNION ALL SELECT {expressions3} FROM {obj3_name}" - - if join_union: - column4 = self.rng.choice(columns_union) - query += f" JOIN {obj4_name} ON {column3} = {column4}" - - if self.rng.choice([True, False]): - query += f" WHERE {expression(Boolean, all_columns_union, self.rng, expr_kind)}" + query += f" {set_op} SELECT {expressions} FROM {obj_name}" + query += join_clause + query += where_clause() + query += group_clause + + # Ordinals keep ORDER BY valid across set operations. Map-typed + # output columns are skipped, same as in the join column selection. + # NOTE: select_types is indexed by output position, which is not + # exprs' indexing once a GROUP BY appends aggregates after the keys. + orderable = [i + 1 for i, dt in enumerate(select_types) if dt != TextTextMap] + if distinct_on_expr is not None: + order_by = [distinct_on_expr] + for i in self.rng.sample(orderable, self.rng.randint(0, len(orderable))): + if i != 1: + order_by.append(str(i)) + query += f" ORDER BY {', '.join(order_by)}" + elif exprs and orderable and self.rng.random() < 0.3: + order_by = [] + for i in self.rng.sample(orderable, self.rng.randint(1, len(orderable))): + direction = self.rng.choice(["", " ASC", " DESC"]) + nulls = self.rng.choice(["", " NULLS FIRST", " NULLS LAST"]) + order_by.append(f"{i}{direction}{nulls}") + query += f" ORDER BY {', '.join(order_by)}" query += f" LIMIT {self.rng.randint(0, 100)}" + if self.rng.random() < 0.2: + query += f" OFFSET {self.rng.randint(0, 100)}" + + if self.rng.random() < 0.15: + query = f"WITH cte0 AS ({query}) SELECT * FROM cte0" return query def exe_prepared(self, query: str, stmt_name: str, exe: Executor) -> None: @@ -460,7 +731,13 @@ def run(self, exe: Executor) -> bool: if self.rng.choice([True, False]) else exe.commit(http=Http.NO) ) + # NOTE: A bounded SUBSCRIBE (UP TO) over an object whose as_of has + # advanced to the end of time (e.g. a finished bounded load generator + # source) soft-panics the optimizer + # (https://linear.app/materializeinc/issue/CLU-169). Left out until that + # is fixed. AS OF AT LEAST 0 below is safe (empty until). query = "SUBSCRIBE " + envelope_used = False if self.rng.choice([True, False]): obj = self.rng.choice(exe.db.db_objects()) query += f"{obj}" @@ -470,12 +747,21 @@ def run(self, exe: Executor) -> bool: columns = self.rng.sample(obj.columns, len(obj.columns)) key = ", ".join(column.name(True) for column in columns) query += f" ENVELOPE {envelope} (KEY ({key}))" - - if self.rng.choice([True, False]): - query += " WITH (SNAPSHOT = false)" + envelope_used = True else: query += f"({self.generate_select_query(exe, ExprKind.MATERIALIZABLE)})" + options = [] + if self.rng.choice([True, False]): + options.append("SNAPSHOT = false") + if not envelope_used and self.rng.random() < 0.3: + options.append("PROGRESS") + if options: + query += f" WITH ({', '.join(options)})" + if self.rng.random() < 0.2: + # AT LEAST always plans, no matter how far the since advanced. + query += " AS OF AT LEAST 0" + exe.execute(f"DECLARE c{self.i} CURSOR FOR {query}", http=Http.NO) while True: rows = self.rng.choice(["ALL", self.rng.randrange(1000)]) @@ -690,6 +976,19 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: # roundtrip can produce NULLs for NOT NULL columns. "violates not-null constraint", "timeout: error trying to connect", + # COPY TO CSV writes a large-year date that COPY FROM CSV then + # fails to parse back. + # See https://linear.app/materializeinc/issue/SS-345 + "expected_dur_like_tokens can only be called with", + # TODO: Reenable when SS-361 is fixed. A COPY FROM without a + # column list plans every target column as its DEFAULT, and + # that projection only acts as identity by accident. Expression + # memoization collapses two identical default literals, so any + # table with two same-type nullable columns (their defaults are + # identical typed nulls) decodes a shifted row. Shifts that are + # type-incompatible fail here, type-compatible ones are + # inserted silently. + "failed to decode Row from a record batch", ] ) if exe.db.complexity == Complexity.DDL: @@ -909,6 +1208,17 @@ def run(self, exe: Executor) -> bool: class InsertReturningAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = super().errors_to_ignore(exe) + # The RETURNING expressions re-render the fully-qualified table and + # column names, so a concurrent schema or table rename landing between + # the INSERT target and the RETURNING clause leaves the two referring to + # different names and the query fails with "does not exist". Same class + # as UpdateAction/DeleteAction. + if exe.db.complexity == Complexity.DDL or exe.db.scenario == Scenario.Rename: + result.extend(["does not exist"]) + return result + def run(self, exe: Executor) -> bool: table = None if exe.insert_table is not None: @@ -972,6 +1282,39 @@ def run(self, exe: Executor) -> bool: return True +class CopyToStdoutAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = super().errors_to_ignore(exe) + result.extend( + [ + "in the same timedomain", + # A prior statement in the read transaction routed it to + # mz_catalog_server, where this COPY of a user object is + # rejected. + 'is not allowed from the "mz_catalog_server" cluster', + # BINARY format is deliberately kept in the mix even though + # some types (e.g. map) have no binary output function. + "no binary output function available for type", + ] + ) + if exe.db.complexity == Complexity.DDL: + result.extend( + [ + "does not exist", + ] + ) + return result + + def run(self, exe: Executor) -> bool: + obj = self.rng.choice(exe.db.db_objects()) + query = f"COPY (SELECT * FROM {obj} LIMIT {self.rng.randint(0, 100)}) TO STDOUT" + if self.rng.choice([True, False]): + format = self.rng.choice(["TEXT", "CSV", "BINARY"]) + query += f" WITH (FORMAT {format})" + exe.copy_to_stdout(query) + return True + + class SourceInsertAction(Action): def run(self, exe: Executor) -> bool: with exe.db.lock: @@ -1048,8 +1391,14 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) - column2 = self.rng.choice(table.columns) - query = f"UPDATE {table} SET {column2.name(True)} = {expression(column2.data_type, table.columns, self.rng, kind=ExprKind.WRITE)} WHERE {expression(Boolean, table.columns, self.rng, kind=ExprKind.WRITE)}" + set_columns = self.rng.sample( + table.columns, self.rng.randint(1, len(table.columns)) + ) + set_clause = ", ".join( + f"{c.name(True)} = {expression(c.data_type, table.columns, self.rng, kind=ExprKind.WRITE)}" + for c in set_columns + ) + query = f"UPDATE {table} SET {set_clause} WHERE {expression(Boolean, table.columns, self.rng, kind=ExprKind.WRITE)}" if self.rng.choice([True, False]): self.stmt_id += 1 self.exe_prepared(query, f"update{self.stmt_id}", exe) @@ -1112,7 +1461,28 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) query = f"DELETE FROM {table}" - if self.rng.random() < 0.95: + using_tables = [ + t + for t in exe.db.tables + if t != table and (not t.temp or t in exe.temp_objects) + ] + # TODO: Drop the RepeatRow gate once database-issues#9308 is fixed. + # DELETE .. USING lowers to a semijoin whose DistinctBy can leave the + # target table with a net-negative row, and every later reader of that + # table then surfaces the corruption. Tolerating that class outside + # RepeatRow would blind the DML and DDL complexities to every genuine + # negative-accumulation finding, so the variant only runs in the one + # scenario that already expects negative multiplicities. + if ( + using_tables + and exe.db.scenario == Scenario.RepeatRow + and self.rng.random() < 0.2 + ): + using_table = self.rng.choice(using_tables) + all_columns = list(table.columns) + list(using_table.columns) + query += f" USING {using_table}" + query += f" WHERE {expression(Boolean, all_columns, self.rng, kind=ExprKind.WRITE)}" + elif self.rng.random() < 0.95: query += f" WHERE {expression(Boolean, table.columns, self.rng, kind=ExprKind.WRITE)}" if self.rng.choice([True, False]): self.stmt_id += 1 @@ -1130,15 +1500,83 @@ def run(self, exe: Executor) -> bool: class CommentAction(Action): - def run(self, exe: Executor) -> bool: - table = self.rng.choice(exe.db.tables) + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "unknown role", + ] + super().errors_to_ignore(exe) - if self.rng.choice([True, False]): - column = self.rng.choice(table.columns) - query = f"COMMENT ON COLUMN {column} IS '{Text.random_value(self.rng)}'" - else: - query = f"COMMENT ON TABLE {table} IS '{Text.random_value(self.rng)}'" + def run(self, exe: Executor) -> bool: + # Only the snapshot needs the global lock. Rendering the candidate + # names is O(tracked objects) with a naughtify each, and holding the + # lock across that serializes every other worker behind this action. + with exe.db.lock: + tables = list(exe.db.tables) + views = list(exe.db.views) + direct_sources = exe.db.kafka_sources + exe.db.webhook_sources + table_sources = ( + exe.db.postgres_sources + + exe.db.mysql_sources + + exe.db.sql_server_sources + ) + loadgen_sources = list(exe.db.loadgen_sources) + sinks = exe.db.kafka_sinks + exe.db.iceberg_sinks + indexes = list(exe.db.indexes) + schemas = list(exe.db.schemas) + dbs = list(exe.db.dbs) + clusters = list(exe.db.clusters) + roles = list(exe.db.roles) + + candidates: list[tuple[str, str]] = [] + for table in tables: + candidates.append(("TABLE", str(table))) + # Columns are only ever appended, never removed, so picking one + # off the snapshot cannot index past the end. + candidates.append(("COLUMN", str(self.rng.choice(table.columns)))) + for view in views: + candidates.append( + ( + "MATERIALIZED VIEW" if view.materialized else "VIEW", + str(view), + ) + ) + # Kafka and webhook source objects are readable directly, the + # others follow the source-table model: str(obj) names the table, + # the ingestion source is a separate catalog item. + for source in direct_sources: + candidates.append(("SOURCE", str(source))) + for source in table_sources: + candidates.append(("TABLE", str(source))) + candidates.append( + ( + "SOURCE", + f"{source.schema}.{identifier(source.executor.source)}", + ) + ) + for source in loadgen_sources: + candidates.append(("TABLE", str(source))) + candidates.append( + ("SOURCE", f"{source.schema}.{identifier(source.source_name())}") + ) + for sink in sinks: + candidates.append(("SINK", str(sink))) + for index in indexes: + candidates.append(("INDEX", str(index))) + for schema in schemas: + candidates.append(("SCHEMA", str(schema))) + for db in dbs: + candidates.append(("DATABASE", str(db))) + for cluster in clusters: + candidates.append(("CLUSTER", str(cluster))) + for role in roles: + candidates.append(("ROLE", str(role))) + candidates.append(("SECRET", "materialize.public.pgpass")) + candidates.append(("CONNECTION", "materialize.public.kafka_conn")) + if not candidates: + return False + kind, name = self.rng.choice(candidates) + comment = self.rng.choice([f"'{Text.random_value(self.rng)}'", "NULL"]) + query = f"COMMENT ON {kind} {name} IS {comment}" exe.execute(query, http=Http.RANDOM) return True @@ -1193,10 +1631,12 @@ def run(self, exe: Executor) -> bool: # The indexed object or its schema may have been dropped # concurrently, taking the index with it. Untrack the index # either way so stale entries don't fill up the set and choke - # off CreateIndexAction. - exe.db.indexes.remove(index) + # off CreateIndexAction. Use discard, not remove: a concurrent + # CASCADE drop's untrack_objects_in_schemas may have already + # removed it, and remove would raise KeyError. + exe.db.indexes.discard(index) raise - exe.db.indexes.remove(index) + exe.db.indexes.discard(index) return True @@ -1260,7 +1700,12 @@ def run(self, exe: Executor) -> bool: query = f"DROP TABLE {table}" exe.execute(query, http=Http.RANDOM) - exe.db.tables.remove(table) + # A concurrent CASCADE drop's untrack_objects_in_schemas may have + # already filtered this table out of the list, tolerate that. + try: + exe.db.tables.remove(table) + except ValueError: + pass return True @@ -1422,11 +1867,28 @@ def run(self, exe: Executor) -> bool: return True -class ReplaceMaterializedViewAction(Action): +class CreateReplacementMaterializedViewAction(Action): + """CREATE REPLACEMENT MATERIALIZED VIEW for a random materialized view. + + Deliberately only creates. The replacement stays alive until an + ApplyReplacementMaterializedViewAction or + DropReplacementMaterializedViewAction finds it in the catalog, from any + worker and possibly only after a kill or 0dt deploy. Resolving the + replacement on the creating connection would tie its lifetime to that + connection, so it could never span an envd restart. Incident 1136 + (bootstrap loses the replacement collection's primary link to the + target's shard, a post-restart DROP of the replacement then finalizes the + target's live shard) lives exactly in that gap. + """ + def errors_to_ignore(self, exe: Executor) -> list[str]: errors = [ - # A concurrent or leaked replacement of the same view + # A concurrent replacement of the same view "because it already has a replacement", + # The target's query constant-folded to a plan without live + # inputs and completed, which can_seal cannot know. The + # SealedCollectionCheckAction oracle arbitrates whether a sealed + # shard is legitimate. "is sealed and thus cannot be replaced", ] + super().errors_to_ignore(exe) if exe.db.scenario == Scenario.Rename: @@ -1437,7 +1899,15 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: def run(self, exe: Executor) -> bool: with exe.db.lock: - mvs = [v for v in exe.db.views if v.materialized] + # Skip views whose shard is known to seal legitimately. Sealing + # is transitive: REFRESH AT views seal after their last refresh, + # repeat_row constant views seal on hydration, and any view that + # reads from a sealing input seals too (View.can_seal). A sealed + # target cannot be replaced, so skipping these avoids wasted + # work. can_seal cannot know about constant-folded queries + # though, so "is sealed" errors are still ignored and + # SealedCollectionCheckAction arbitrates damage. + mvs = [v for v in exe.db.views if v.materialized and not v.can_seal] if not mvs: return False view = self.rng.choice(mvs) @@ -1453,17 +1923,298 @@ def run(self, exe: Executor) -> bool: exe.execute( f"CREATE REPLACEMENT MATERIALIZED VIEW {tmp_mv} FOR {view} AS {view.get_select()}", ) - time.sleep(self.rng.random()) + return True + + +class ResolveReplacementMaterializedViewAction(Action): + """Base for actions resolving live replacements. + + Replacements are picked from the catalog instead of tracked state so that + replacements created by other workers or left behind by dead connections, + kills, and 0dt deploys are found too. + """ + + def pick_replacement(self, exe: Executor) -> tuple[str, str] | None: + exe.execute( + "SELECT rd.name, rs.name, r_mv.name, td.name, ts.name, t_mv.name" + " FROM mz_internal.mz_replacements r" + " JOIN mz_catalog.mz_materialized_views r_mv ON r_mv.id = r.id" + " JOIN mz_catalog.mz_schemas rs ON rs.id = r_mv.schema_id" + " JOIN mz_catalog.mz_databases rd ON rd.id = rs.database_id" + " JOIN mz_catalog.mz_materialized_views t_mv ON t_mv.id = r.target_id" + " JOIN mz_catalog.mz_schemas ts ON ts.id = t_mv.schema_id" + " JOIN mz_catalog.mz_databases td ON td.id = ts.database_id", + http=Http.NO, + ) + rows = exe.cur.fetchall() + if not rows: + return None + r_db, r_schema, r_name, t_db, t_schema, t_name = self.rng.choice(rows) + replacement = f"{identifier(r_db)}.{identifier(r_schema)}.{identifier(r_name)}" + target = f"{identifier(t_db)}.{identifier(t_schema)}.{identifier(t_name)}" + return replacement, target + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + # Reading catalog relations inside a txn that already touched + # user objects crosses timedomains. + "in the same timedomain", + ] + super().errors_to_ignore(exe) + + +class ApplyReplacementMaterializedViewAction(ResolveReplacementMaterializedViewAction): + def errors_to_ignore(self, exe: Executor) -> list[str]: + errors = [ + # A concurrent apply or drop won the race for this replacement + "replacement materialized view does not exist", + # The target's shard sealed, which is legitimate for targets + # whose query constant-folded to a plan without live inputs. + # SealedCollectionCheckAction arbitrates whether a sealed shard + # is damage, so it is not treated as a failure here. + "is sealed and thus cannot be replaced", + ] + super().errors_to_ignore(exe) + if exe.db.scenario == Scenario.Rename: + # The target was renamed between picking and applying + errors += ["does not exist"] + return errors + + def run(self, exe: Executor) -> bool: + picked = self.pick_replacement(exe) + if picked is None: + return False + replacement, target = picked + # APPLY REPLACEMENT waits for the target's write frontier to catch up + # to the replacement's. The wait never finishes if the target is + # concurrently cascade-dropped (database-issues#9820), so bound it + # with a statement timeout. + exe.execute("SET statement_timeout = '60s'") + # A timeout is expected for as long as this one is configured. + exe.statement_timeout_set = True try: - exe.execute(f"ALTER MATERIALIZED VIEW {view} APPLY REPLACEMENT {tmp_mv}") - except QueryError: - # Clean up, a leaked replacement blocks all future replacements - # of this view. + exe.execute( + f"ALTER MATERIALIZED VIEW {target} APPLY REPLACEMENT {replacement}" + ) + finally: try: - exe.execute(f"DROP MATERIALIZED VIEW IF EXISTS {tmp_mv}") + exe.execute("RESET statement_timeout") except QueryError: + # The timeout is still configured, so keep tolerating it. + pass + else: + # Back at the default timeout, so a timeout is a genuine + # finding again. The worker's statements are sequential, + # nothing from the window above can still be in flight. + exe.statement_timeout_set = False + return True + + +class DropReplacementMaterializedViewAction(ResolveReplacementMaterializedViewAction): + """Drop a live replacement without applying it. + + This is the incident 1136 trigger statement when it runs after an envd + restart: bootstrap loses the replacement collection's primary link, so + the drop wrongly finalizes the target's live shard. + SealedCollectionCheckAction detects the resulting damage. + """ + + def run(self, exe: Executor) -> bool: + picked = self.pick_replacement(exe) + if picked is None: + return False + replacement, _ = picked + # IF EXISTS, a concurrent apply or drop may have resolved it already. + exe.execute(f"DROP MATERIALIZED VIEW IF EXISTS {replacement}") + return True + + +def plan_user_input_ids(plan: str) -> list[str]: + """User collection ids that an `EXPLAIN .. AS JSON` dataflow plan reads. + + Raises on a plan whose shape it cannot read, rather than returning an empty + list. Constant folding legitimately produces plans that read nothing, so + "reads nothing" and "no longer understands the plan" are indistinguishable + to a caller, and treating the second as the first retires whatever the + caller checks without a trace. Only accepts dataflow (MIR) plans, a + fast-path peek plan carries its inputs elsewhere and is rejected. + """ + try: + parsed = json.loads(plan) + except json.JSONDecodeError as e: + raise ValueError(f"EXPLAIN .. AS JSON output does not parse ({e}): {plan}") + + # `Constant` and `Get` are the only leaves of MirRelationExpr, so a plan + # with neither is not a shape this can read. Global ids outside `Get` are + # not inputs, e.g. the index an `access_strategy` picked. + get_ids: list = [] + constant = False + + def walk(node: object) -> None: + nonlocal constant + if isinstance(node, dict): + get = node.get("Get") + if isinstance(get, dict) and "id" in get: + get_ids.append(get["id"]) + if "Constant" in node: + constant = True + for value in node.values(): + walk(value) + elif isinstance(node, list): + for value in node: + walk(value) + + walk(parsed) + if not get_ids and not constant: + raise ValueError(f"plan has no Get or Constant leaf: {plan}") + + user_ids = set() + for get_id in get_ids: + match get_id: + case {"Global": {"User": int(gid)}}: + user_ids.add(f"u{gid}") + case {"Local": _} | { + "Global": {"System": _} + | {"Transient": _} + | {"IntrospectionSourceIndex": _} + }: + # A Let binding or a non-user namespace, neither is a user + # collection with a frontier to check. pass + case _: + raise ValueError(f"unrecognized Get id {get_id!r} in plan: {plan}") + return sorted(user_ids) + + +class SealedCollectionCheckAction(Action): + """Client-side data-loss oracle for wrongly finalized persist shards. + + A shard's upper only becomes empty when the shard is sealed. That is + legitimate when the writing dataflow completed (a constant plan, inputs + that all sealed, a REFRESH AT view past its last refresh, a bounded load + generator that finished) and destructive when shard finalization ran + against a live collection. Incident 1136 is the known destructive + trigger: dropping a replacement after an envd restart finalizes the + target's shard because bootstrap dropped the replacement collection's + primary link. Tables are covered too because versioned tables + (ALTER TABLE ADD COLUMN) carry the same primary ownership chain. + + A live user table or materialized view with an empty write frontier and + at least one live plan input has no legitimate explanation and is + reported as damage. This oracle is the single arbiter of wrongly sealed + shards, the replacement actions ignore "is sealed" errors instead of + treating them as failures. + """ + + def applicable(self, exe: Executor) -> bool: + # repeat_row constant views seal legitimately on hydration and are + # indistinguishable from damage in the catalog. + return exe.db.scenario != Scenario.RepeatRow + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "in the same timedomain", + ] + super().errors_to_ignore(exe) + + def completed_legitimately( + self, exe: Executor, database: str, schema: str, name: str + ) -> bool: + """Whether a sealed materialized view's dataflow completed on its own. + + A dataflow completes, sealing its shard, once it can never produce + more output: its optimized plan reads no storage collections at all + (constant folding, e.g. a join ON false, drops inputs the catalog + still records as dependencies), or every storage collection it reads + is itself sealed. Only a sealed view with at least one live plan + input is damage. + """ + try: + exe.execute( + "EXPLAIN OPTIMIZED PLAN AS JSON FOR MATERIALIZED VIEW" + f" {identifier(database)}.{identifier(schema)}.{identifier(name)}", + http=Http.NO, + ) + plan = "\n".join(str(row[0]) for row in exe.cur.fetchall()) + except QueryError as e: + if "unknown catalog item" in str(e) or "does not exist" in str(e): + # Dropped concurrently, so its shard is allowed to seal. + return True raise + input_ids = plan_user_input_ids(plan) + if not input_ids: + return True + id_list = ", ".join(f"'{i}'" for i in input_ids) + exe.execute( + "SELECT count(*) FROM mz_internal.mz_frontiers" + f" WHERE object_id IN ({id_list}) AND write_frontier IS NOT NULL", + http=Http.NO, + ) + return exe.cur.fetchall()[0][0] == 0 + + def run(self, exe: Executor) -> bool: + # Sealing is transitive: a view reading from a REFRESH AT view seals + # once that input seals, even though its own refresh strategy is not + # 'at', and likewise for anything reading a bounded load generator's + # table. Walk the dependency graph up from every REFRESH AT view and + # every load generator source and exclude everything that transitively + # depends on one. The COUNTER sources are UP TO bounded and seal when + # they finish. The multi-subsource generators never seal, so sweeping + # them in only widens the exclusion, erring toward tolerating. repeat_row + # constant views also seal transitively but are not marked in the + # catalog, which is why this oracle is disabled in the RepeatRow + # scenario, see applicable. Plain user tables never seal and need no + # exclusions, and source-fed tables are covered by the walk through + # their source. + def sealed_collections() -> list: + exe.execute( + "WITH MUTUALLY RECURSIVE" + " sealing(id text) AS (" + "SELECT rs.materialized_view_id" + " FROM mz_internal.mz_materialized_view_refresh_strategies rs" + " WHERE rs.type = 'at'" + " UNION" + " SELECT s.id" + " FROM mz_catalog.mz_sources s" + " WHERE s.type = 'load-generator'" + " UNION" + " SELECT d.object_id" + " FROM mz_internal.mz_object_dependencies d" + " JOIN sealing s ON d.referenced_object_id = s.id)" + " SELECT o.id, d.name, sc.name, o.name, o.type" + " FROM mz_catalog.mz_objects o" + " JOIN mz_catalog.mz_schemas sc ON sc.id = o.schema_id" + " JOIN mz_catalog.mz_databases d ON d.id = sc.database_id" + " JOIN mz_internal.mz_frontiers f ON f.object_id = o.id" + " WHERE o.type IN ('materialized-view', 'table')" + " AND o.id LIKE 'u%'" + " AND f.write_frontier IS NULL" + " AND o.id NOT IN (SELECT id FROM sealing)", + http=Http.NO, + ) + return exe.cur.fetchall() + + sealed = sealed_collections() + if not sealed: + return True + # The catalog relations and mz_frontiers are not updated atomically, + # so a query timestamp can land inside a concurrent CREATE or DROP + # where the object is cataloged but its frontier reads as the empty + # antichain. A wrongly finalized shard is permanent, so only report + # objects that are still sealed on a re-check. + time.sleep(5) + sealed_ids = {row[0] for row in sealed} + damaged = [ + row + for row in sealed_collections() + if row[0] in sealed_ids + and not ( + row[4] == "materialized-view" + and self.completed_legitimately(exe, row[1], row[2], row[3]) + ) + ] + if damaged: + raise ValueError( + "Collections with wrongly sealed persist shards" + f" (incident 1136 class): {damaged}" + ) return True @@ -1492,13 +2243,13 @@ def run(self, exe: Executor) -> bool: # Iceberg sinks always have a key, so only allow a conservative # case: all names, types, and nullabilities match, which also # guarantees the key columns exist in the new object. - # TODO: Switch back when SS-324 is fixed to make sure it errors + # TODO: Switch back when SS-344 is fixed to make sure it errors # instead of causing a stall objs = [] old_cols = { c.name(True): (c.data_type, c.nullable) for c in old_object.columns } - for o in exe.db.db_objects_without_views(): + for o in exe.db.db_objects_for_sinks(): if isinstance(old_object, WebhookSource): continue if isinstance(o, WebhookSource): @@ -1554,14 +2305,14 @@ def run(self, exe: Executor) -> bool: # single column formats objs = [ o - for o in exe.db.db_objects_without_views() + for o in exe.db.db_objects_for_sinks() if len(o.columns) == 1 and o.columns[0].data_type == old_object.columns[0].data_type ] elif sink.format in ["FORMAT JSON"]: # We should be able to format all data types as JSON, and they have no # particular backwards-compatiblility requirements. - objs = [o for o in exe.db.db_objects_without_views()] + objs = [o for o in exe.db.db_objects_for_sinks()] else: # Avro schema migration checking can be quite strict, and we need to be not only # compatible with the latest object's schema but all previous schemas. @@ -1574,7 +2325,7 @@ def run(self, exe: Executor) -> bool: old_cols = { c.name(True): (c.data_type, c.nullable) for c in old_object.columns } - for o in exe.db.db_objects_without_views(): + for o in exe.db.db_objects_for_sinks(): if isinstance(old_object, WebhookSource): continue if isinstance(o, WebhookSource): @@ -1802,43 +2553,241 @@ def run(self, exe: Executor) -> bool: return True -class CommitRollbackAction(Action): - def run(self, exe: Executor) -> bool: - if not exe.action_run_since_last_commit_rollback: - return False +class ParameterizedQueryAction(Action): + """PREPARE a query with $1..$n placeholders, then EXECUTE it with values. - if self.rng.random() < 0.7: - exe.commit() - else: - exe.rollback() - exe.action_run_since_last_commit_rollback = False + Exercises the parameter-type-inference and bind/assignment-cast path that + the workload's other prepared statements (no parameters) never reach.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = super().errors_to_ignore(exe) + result.extend( + [ + "in the same timedomain", + 'is not allowed from the "mz_catalog_server" cluster', + ] + ) + if exe.db.complexity == Complexity.DDL: + result.extend(["does not exist"]) + return result + + def run(self, exe: Executor) -> bool: + obj = self.rng.choice(exe.db.db_objects()) + n = self.rng.randint(1, 4) + # Record and record list are expression-only pseudo types, a + # parameter cast to them fails with "cannot reference pseudo type". + param_types = [self.rng.choice(list(DATA_TYPES_FOR_COLUMNS)) for _ in range(n)] + projection = ", ".join( + f"${i + 1}::{t.name()}" for i, t in enumerate(param_types) + ) + self.stmt_id += 1 + name = f"pq{self.stmt_id}" + query = f"SELECT {projection} FROM {obj} LIMIT {self.rng.randint(0, 10)}" + # Each argument is cast to its parameter's declared type so the + # assignment cast on EXECUTE always succeeds (e.g. a bytea parameter + # rejects a bare text literal). + values = ", ".join( + f"({t.random_value(self.rng, in_query=True)})::{t.name()}" + for t in param_types + ) + # Run sequentially, not in a try/finally: if EXECUTE fails it aborts + # the transaction, and a DEALLOCATE in a finally would then fail with + # "current transaction is aborted", masking the real error. On failure + # the statement leaks on the session, which is harmless: statement + # names are never reused, and the session's prepared statements die + # with it. + exe.execute(f"PREPARE {name} AS {query}", http=Http.NO) + exe.execute(f"EXECUTE {name} ({values})", http=Http.NO, fetch=True) + exe.execute(f"DEALLOCATE {name}", http=Http.NO) return True -class FlipFlagsAction(Action): - def __init__( - self, - rng: random.Random, - composition: Composition | None, - ): - super().__init__(rng, composition) +class BoundedStalenessReadAction(Action): + """A read under `bounded staleness` isolation, a distinct timestamp- + selection path that never blocks and returns 40001 when the freshness + bound cannot be met. The isolation is set transiently around the read and + restored afterwards, since bounded staleness is read-only and would break + writes.""" - BOOLEAN_FLAG_VALUES = ["TRUE", "FALSE"] + def applicable(self, exe: Executor) -> bool: + return exe.db.flags.get("enable_bounded_staleness_isolation", "FALSE") == "TRUE" - self.flags_with_values: dict[str, list[str]] = dict() - self.flags_with_values["persist_blob_target_size"] = ( - # 1 MiB, 16 MiB, 128 MiB - ["1048576", "16777216", "134217728"] - ) - for flag in ["catalog", "source", "snapshot", "txn"]: - self.flags_with_values[f"persist_use_critical_since_{flag}"] = ( - BOOLEAN_FLAG_VALUES - ) - self.flags_with_values["persist_claim_unclaimed_compactions"] = ( - BOOLEAN_FLAG_VALUES + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = super().errors_to_ignore(exe) + result.extend( + [ + # The flag was flipped off between applicable() and run(). + "is not available", + # The freshness bound could not be met. Bounded staleness + # never blocks, it errors instead. + "not been materialized", + "could not find a valid timestamp for the query", + "cannot serve query under bounded staleness", + # A leaked real_time_recency SET on the session (its own reset + # was discarded on a prior query's error path) conflicts with + # bounded staleness. + "cannot be combined with bounded staleness", + "in the same timedomain", + 'is not allowed from the "mz_catalog_server" cluster', + ] ) - self.flags_with_values["persist_optimize_ignored_data_fetch"] = ( - BOOLEAN_FLAG_VALUES + if exe.db.complexity == Complexity.DDL: + result.extend(["does not exist"]) + return result + + def run(self, exe: Executor) -> bool: + bound = self.rng.choice(["1s", "5s", "30s"]) + restore = f"SET TRANSACTION_ISOLATION TO '{exe.isolation}'" + exe.execute( + f"SET TRANSACTION_ISOLATION TO 'bounded staleness {bound}'", + explainable=False, + ) + try: + query = self.generate_select_query(exe, ExprKind.ALL) + exe.execute(query, http=Http.NO, fetch=True) + except QueryError: + # The restore must not raise on top of the read's error, e.g. on a + # killed or cancelled session, that error would replace the read's + # and hide the real failure. A leaked bounded staleness isolation + # makes later actions fail with errors only this action ignores, so + # a failed restore forces a reconnect rather than being swallowed. + try: + exe.execute(restore, explainable=False) + except QueryError: + exe.reconnect_next = True + raise + # No error is being masked here, so a failing restore is a genuine + # failure and propagates. + exe.execute(restore, explainable=False) + return True + + +class ReadOnlyTransactionAction(Action): + """A multi-statement `BEGIN READ ONLY` transaction. All reads run at one + pinned timestamp (one timedomain), and holding the read pins compaction + for the transaction's lifetime.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = super().errors_to_ignore(exe) + result.extend( + [ + "in the same timedomain", + 'is not allowed from the "mz_catalog_server" cluster', + ] + ) + if exe.db.complexity == Complexity.DDL: + result.extend(["does not exist"]) + return result + + def run(self, exe: Executor) -> bool: + isolation = self.rng.choice( + [ + "", + " ISOLATION LEVEL SERIALIZABLE", + " ISOLATION LEVEL STRICT SERIALIZABLE", + ] + ) + exe.execute(f"BEGIN{isolation} READ ONLY", http=Http.NO) + try: + for _ in range(self.rng.randint(1, 3)): + query = self.generate_select_query(exe, ExprKind.ALL) + exe.execute(query, http=Http.NO, fetch=True) + except QueryError: + # The transaction still has to end, an open one wedges every later + # action on this session, but ending it must not raise over the + # read's error and hide the real failure. COMMIT on a failed + # transaction is turned into a rollback by the coordinator anyway, + # so always rolling back here loses no coverage. + try: + exe.execute("ROLLBACK", http=Http.NO) + except QueryError: + exe.reconnect_next = True + raise + # Outside the error path both endings are worth exercising, and a + # failure of the end statement itself is a genuine one. + end = "COMMIT" if self.rng.choice([True, False]) else "ROLLBACK" + exe.execute(end, http=Http.NO) + return True + + +class DDLTransactionAction(Action): + """A DDL statement inside an explicit `BEGIN`/`COMMIT`. Materialize allows + only a single statement in a DDL transaction, so the value over autocommit + DDL is the open commit window: racing concurrent DDL against it exercises + the "another session modified the catalog while this DDL transaction was + open" serialization path.""" + + def applicable(self, exe: Executor) -> bool: + return exe.db.complexity in (Complexity.DDL, Complexity.DDLOnly) + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "another session modified the catalog", + "unknown schema", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + if len([t for t in exe.db.tables if not t.temp]) >= MAX_TABLES: + return False + try: + schema = self.rng.choice(exe.db.schemas) + except IndexError: + return False + table_id = exe.db.table_id + exe.db.table_id += 1 + table = Table(self.rng, table_id, schema) + exe.execute("BEGIN", http=Http.NO) + try: + table.create(exe) + exe.execute("COMMIT", http=Http.NO) + except QueryError: + try: + exe.execute("ROLLBACK", http=Http.NO) + except QueryError: + pass + raise + with exe.db.lock: + exe.db.tables.append(table) + return True + + +class CommitRollbackAction(Action): + def run(self, exe: Executor) -> bool: + if not exe.action_run_since_last_commit_rollback: + return False + + if self.rng.random() < 0.7: + exe.commit() + else: + exe.rollback() + exe.action_run_since_last_commit_rollback = False + return True + + +class FlipFlagsAction(Action): + def __init__( + self, + rng: random.Random, + composition: Composition | None, + ): + super().__init__(rng, composition) + + BOOLEAN_FLAG_VALUES = ["TRUE", "FALSE"] + + self.flags_with_values: dict[str, list[str]] = dict() + self.flags_with_values["persist_blob_target_size"] = ( + # 1 MiB, 16 MiB, 128 MiB + ["1048576", "16777216", "134217728"] + ) + for flag in ["catalog", "source", "snapshot", "txn"]: + self.flags_with_values[f"persist_use_critical_since_{flag}"] = ( + BOOLEAN_FLAG_VALUES + ) + self.flags_with_values["persist_claim_unclaimed_compactions"] = ( + BOOLEAN_FLAG_VALUES + ) + self.flags_with_values["persist_optimize_ignored_data_fetch"] = ( + BOOLEAN_FLAG_VALUES ) self.flags_with_values["persist_source_fetch_concurrency"] = [ "1", @@ -1957,6 +2906,9 @@ def __init__( BOOLEAN_FLAG_VALUES ) self.flags_with_values["enable_alter_table_add_column"] = BOOLEAN_FLAG_VALUES + self.flags_with_values["enable_bounded_staleness_isolation"] = ( + BOOLEAN_FLAG_VALUES + ) self.flags_with_values["enable_arrangement_dictionary_compression_alpha"] = ( BOOLEAN_FLAG_VALUES ) @@ -1972,10 +2924,10 @@ def __init__( BOOLEAN_FLAG_VALUES ) self.flags_with_values["cluster"] = ["quickstart", "dont_exist"] - self.flags_with_values["enable_frontend_peek_sequencing"] = [ - "true", - "false", - ] + # NOTE: enable_frontend_peek_sequencing is pinned off in + # ADDITIONAL_SYSTEM_PARAMETER_DEFAULTS (frontend-peek read-hold vs + # compaction race, https://linear.app/materializeinc/issue/SQL-520), so + # it is not flipped here. self.flags_with_values["enable_frontend_subscribes"] = [ "true", "false", @@ -1992,11 +2944,17 @@ def __init__( self.flags_with_values["enable_column_paged_batcher_spill"] = ( BOOLEAN_FLAG_VALUES ) + # Fractions of the *cgroup* memory limit, which under mzcompose's + # process orchestrator is the whole container budget shared by + # environmentd and every replica, not one replica's allowance. A + # process-singleton pool at 0.25 is 6 GiB per clusterd on the 24 GiB CI + # agent, so the values stay small enough that all replicas together + # cannot claim the budget. Small pools also fill up sooner, which + # exercises the paging and compression paths more, not less. self.flags_with_values["column_paged_batcher_budget_fraction"] = [ "0.0", "0.01", - "0.05", - "0.25", + "0.02", ] self.flags_with_values["column_paged_batcher_lz4"] = BOOLEAN_FLAG_VALUES self.flags_with_values["column_paged_batcher_swap_pageout"] = ( @@ -2010,10 +2968,14 @@ def __init__( self.flags_with_values["column_paged_batcher_eager_backing"] = ( BOOLEAN_FLAG_VALUES ) + # Same shared-budget reasoning as the budget fraction above. Sharing its + # value set keeps all three orderings against the budget covered: a + # target of 0 collapses the compressed tier, a target below the budget + # pages out above it, and a target above the budget pages nothing out. self.flags_with_values["column_paged_batcher_pool_rss_target_fraction"] = [ "0.0", - "0.25", - "0.5", + "0.01", + "0.02", ] self.flags_with_values["enable_upsert_paged_spill"] = BOOLEAN_FLAG_VALUES # 0 forces the estimated-size path for every table, the default forces @@ -2326,12 +3288,21 @@ def run(self, exe: Executor) -> bool: flag_value = self.rng.choice(self.flags_with_values[flag_name]) + # Occasionally restore the default instead, a distinct path from + # SET-to-value. + reset = self.rng.random() < 0.1 + conn = None try: conn = self.create_system_connection(exe) if cluster is not None: self.set_cluster_compression(conn, cluster) + elif reset: + self.reset_flag(conn, flag_name) + # Gates reading exe.db.flags fall back to their conservative + # default when the key is absent. + exe.db.flags.pop(flag_name, None) else: self.flip_flag(conn, flag_name, flag_value) exe.db.flags[flag_name] = flag_value @@ -2363,6 +3334,12 @@ def set_cluster_compression(self, conn: Connection, cluster: Cluster) -> None: with conn.cursor() as cur: cur.execute(f"ALTER CLUSTER {cluster} {option};".encode()) + def reset_flag(self, conn: Connection, flag_name: str) -> None: + with conn.cursor() as cur: + cur.execute( + f"ALTER SYSTEM RESET {flag_name};".encode(), + ) + class CreateViewAction(Action): def errors_to_ignore(self, exe: Executor) -> list[str]: @@ -2466,7 +3443,46 @@ def run(self, exe: Executor) -> bool: else: query = f"DROP VIEW {view}" exe.execute(query, http=Http.RANDOM) - exe.db.views.remove(view) + # A concurrent CASCADE drop's untrack_objects_in_schemas may have + # already filtered this view out of the list, tolerate that. + try: + exe.db.views.remove(view) + except ValueError: + pass + return True + + +class CreateOrReplaceViewAction(Action): + """In-place swap of an existing view's definition via CREATE OR REPLACE. + + The body is unchanged, so dependents stay valid, but the coordinator still + tears down and rebuilds the item (and the dataflow, for a materialized + view). Racing that swap against reads and concurrent DDL is the point.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + errors = [ + # A dependent references a column the replacement would drop. The + # body is unchanged here, but a concurrent replacement may have + # changed it. + "still depended upon by", + "replica-targeted materialized views is not supported", + "unknown cluster replica", + ] + super().errors_to_ignore(exe) + if exe.db.scenario == Scenario.Rename: + # A base object was renamed, invalidating the captured body. + errors += ["does not exist", "ambiguous reference to schema name"] + return errors + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + views = [view for view in exe.db.views if not view.temp] + if not views: + return False + view = self.rng.choice(views) + with view.lock: + if view not in exe.db.views: + return False + view.create(exe, or_replace=True) return True @@ -2514,6 +3530,48 @@ def run(self, exe: Executor) -> bool: return True +class AlterRoleAction(Action): + """ALTER ROLE ... SET / RESET a default session variable. + + Exercises the per-role session-default path (applied at session init) and + role-name resolution on the ALTER path, neither of which CREATE/DROP ROLE + touch. Part of broadening ALTER coverage (the ALTER paths are where several + catalog bugs have hidden vs the well-worn CREATE/DROP).""" + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if not exe.db.roles: + return False + role = self.rng.choice(exe.db.roles) + with role.lock: + # Was dropped while we were acquiring the lock. + if role not in exe.db.roles: + return False + var, value = self.rng.choice( + [ + ("cluster", "'quickstart'"), + ("transaction_isolation", "'serializable'"), + ("statement_timeout", "'120s'"), + ("search_path", "public"), + ] + ) + query = ( + f"ALTER ROLE {role} RESET {var}" + if self.rng.choice([True, False]) + else f"ALTER ROLE {role} SET {var} = {value}" + ) + try: + exe.execute(query, http=Http.RANDOM) + except QueryError as e: + # Concurrent DROP ROLE, expected as with DropRoleAction. + if ( + exe.db.scenario not in (Scenario.Kill, Scenario.ZeroDowntimeDeploy) + or "unknown role" not in e.msg + ): + raise e + return True + + class CreateClusterAction(Action): def run(self, exe: Executor) -> bool: with exe.db.lock: @@ -2524,9 +3582,7 @@ def run(self, exe: Executor) -> bool: cluster = Cluster( cluster_id, managed=self.rng.choice([True, False]), - size=self.rng.choice( - ["scale=1,workers=1", "scale=1,workers=4", "scale=2,workers=2"] - ), + size=self.rng.choice(["scale=1,workers=1", "scale=1,workers=2"]), replication_factor=self.rng.choice([1, 2]), introspection_interval="1s", ) @@ -2678,9 +3734,7 @@ def run(self, exe: Executor) -> bool: replica = ClusterReplica( replica_id, - size=self.rng.choice( - ["scale=1,workers=1", "scale=1,workers=4", "scale=2,workers=2"] - ), + size=self.rng.choice(["scale=1,workers=1", "scale=1,workers=2"]), cluster=cluster, ) replica.create(exe) @@ -2726,8 +3780,10 @@ def run(self, exe: Executor) -> bool: class ReconfigureClusterAction(Action): - """Gracefully resize a random managed cluster and assert convergence. + """Gracefully resize a random managed cluster and watch its record settle. + The asserted invariant is that a reconfiguration record never sits + in-progress past its own deadline, whether it cuts over or rolls back. Regression coverage for SQL-530: readiness must not wait for single-replica sources (Postgres, MySQL, SQL Server) hosted on the cluster, which never hydrate on the reconfiguration's target replicas @@ -2778,40 +3834,85 @@ def run(self, exe: Executor) -> bool: # e.g. when another worker put a REFRESH EVERY materialized view # with a far-future refresh on the cluster, which cannot hydrate # on the target replicas before its refresh time. What must never - # happen is the record staying "in-progress" past its deadline, - # which is how a wedged readiness check (SQL-530) manifests. Only - # trust a terminal status once the ALTER's own deadline has - # passed: builtin table reads can lag the catalog, so before that - # a terminal status can only be a stale record from an earlier - # reconfiguration of this cluster. + # happen is a record staying "in-progress" past its own deadline, + # which is how a wedged readiness check (SQL-530) manifests. + # + # `mz_now()` is the timestamp the read itself ran at, and comparing + # it against the record's `deadline` is what makes that assertion + # sound. `mz_clusters` is a builtin table and + # `mz_cluster_reconfigurations` a builtin materialized view + # maintained by mz_catalog_server, and a SERIALIZABLE read of either + # takes the freshest timestamp that does not block, so under load it + # can lag wall clock by minutes. Judging the deadline against the + # test's wall clock therefore calls a correctly rolled-back record + # wedged as soon as the read lags. Both sides coming from the same + # read keeps them in the same frame, stale or not. name_literal = cluster.name().replace("'", "''") query = ( - "SELECT c.size, r.status FROM mz_clusters c " + "SELECT c.size, r.status, r.deadline::text, mz_now()::text " + "FROM mz_clusters c " "LEFT JOIN mz_internal.mz_cluster_reconfigurations r ON r.cluster_id = c.id " f"WHERE c.name = '{name_literal}'" ) - status = None + # The deadline our own ALTER wrote is `now() + 120s` in the same + # millisecond epoch `mz_now()` reports, so a read past this is + # guaranteed to show our record or a later one, never a stale + # terminal record from an earlier reconfiguration of this cluster. + our_deadline_ms = int((alter_started + 120) * 1000) seen = False - deadline = time.time() + 180 - while time.time() < deadline: + # Must stay comfortably under the 300s the run gives all workers to + # join after `end_time` (`parallel_workload.run`). A poll that + # outlives that budget makes a worker doing its job look wedged, and + # the run then hard-exits 0 without its final checks. + poll_until = time.time() + 240 + while time.time() < poll_until: exe.execute(query) rows = exe.cur.fetchall() if not rows: # `mz_clusters` is a builtin table, so the read can lag a # concurrent rename or swap of this cluster and not show - # the polled name yet. Keep polling and let the deadline + # the polled name yet. Keep polling and let the check # below decide, rather than indexing an empty result. time.sleep(1) continue seen = True - size, status = rows[0] + size, status, record_deadline, read_ts = rows[0] + read_ts_ms = int(read_ts) if size == new_size: cluster.size = new_size return True - if time.time() > alter_started + 120 and status in ( - "timed-out", - "resource-exhausted", + # 60s of slack over the controller's 5s tick, which is what + # turns a reached deadline into the terminal status. + if ( + status == "in-progress" + and read_ts_ms > int(record_deadline) + 60_000 ): + raise ValueError( + f"Graceful reconfiguration of cluster {cluster} to size " + f"{new_size} sat in-progress {read_ts_ms - int(record_deadline)}ms " + f"past its deadline {record_deadline}" + ) + if read_ts_ms > our_deadline_ms and status is None: + raise ValueError( + f"Reconfiguration record of cluster {cluster} is gone as of " + f"{read_ts_ms}, past the deadline {our_deadline_ms} of the " + f"reconfiguration to size {new_size} that wrote it" + ) + # Any settled record is a legitimate end state, so stop on every + # status but "in-progress". A rollback at the deadline + # ("timed-out"), a controller that could not get the replicas + # ("resource-exhausted"), a concurrent shape-touching ALTER + # calling the transition off ("cancelled"), or one that + # retargeted it to a shape that then cut over ("finalized" on a + # size that is not ours) all mean the reconfiguration settled. + # FlipFlagsAction alters EXPERIMENTAL ARRANGEMENT COMPRESSION + # without holding the cluster lock, and that is a shape + # dimension (`alter_changes_replica_shape`), so it reaches the + # record this poll is watching. Once ours settles, such an ALTER + # even starts a fresh record, in-progress with the 24h default + # deadline, which is why running out of the budget below on an + # in-progress record is not by itself a failure. + if read_ts_ms > our_deadline_ms and status != "in-progress": return True time.sleep(1) if not seen: @@ -2820,10 +3921,7 @@ def run(self, exe: Executor) -> bool: f"name the reconfiguration to size {new_size} polled for, so the " f"name the workload holds does not match the catalog" ) - raise ValueError( - f"Graceful reconfiguration of cluster {cluster} to size {new_size} " - f"did not complete (reconfiguration status: {status})" - ) + return True class GrantPrivilegesAction(Action): @@ -2832,7 +3930,7 @@ def run(self, exe: Executor) -> bool: if not exe.db.roles: return False role = self.rng.choice(exe.db.roles) - privilege = self.rng.choice(["SELECT", "INSERT", "UPDATE", "ALL"]) + privilege = self.rng.choice(["SELECT", "INSERT", "UPDATE", "DELETE", "ALL"]) tables_views: list[DBObject] = [*exe.db.tables, *exe.db.views] table = self.rng.choice(tables_views) with table.lock, role.lock: @@ -2860,7 +3958,7 @@ def run(self, exe: Executor) -> bool: if not exe.db.roles: return False role = self.rng.choice(exe.db.roles) - privilege = self.rng.choice(["SELECT", "INSERT", "UPDATE", "ALL"]) + privilege = self.rng.choice(["SELECT", "INSERT", "UPDATE", "DELETE", "ALL"]) tables_views: list[DBObject] = [*exe.db.tables, *exe.db.views] table = self.rng.choice(tables_views) with table.lock, role.lock: @@ -2882,68 +3980,597 @@ def run(self, exe: Executor) -> bool: return True -# TODO: Should factor this out so can easily use it without action -class ReconnectAction(Action): - def __init__( - self, - rng: random.Random, - composition: Composition | None, - random_role: bool = True, - ): - super().__init__(rng, composition) - self.random_role = random_role +class GrantRoleAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "unknown role", + # Concurrent memberships can close a cycle, which is rejected. + "is a member of role", + ] + super().errors_to_ignore(exe) def run(self, exe: Executor) -> bool: - exe.mz_service = "materialized" - exe.log("reconnecting") - # The connection's temp objects die with it, drop them from the - # tracked state so other workers stop querying them. - if exe.temp_objects: - with exe.db.lock: - exe.db.tables[:] = [ - t for t in exe.db.tables if t not in exe.temp_objects - ] - exe.db.views[:] = [v for v in exe.db.views if v not in exe.temp_objects] - exe.temp_objects.clear() - host = exe.db.host + with exe.db.lock: + if len(exe.db.roles) < 2: + return False + role1, role2 = self.rng.sample(exe.db.roles, 2) + # Both locks are held across the round trip below, so take them in + # a fixed order. Two workers that sampled the same pair in + # opposite orders would otherwise deadlock. The GRANT operands + # keep the sampled order, so memberships still form in both + # directions, including the cycles this action provokes. + first, second = sorted((role1, role2), key=lambda role: role.role_id) + with first.lock, second.lock: + if role1 not in exe.db.roles or role2 not in exe.db.roles: + return False + exe.execute(f"GRANT {role1} TO {role2}", http=Http.RANDOM) + return True - def pg_port() -> int: - # System workers (e.g. the Cancel worker) live on the internal - # port, everyone else on the external one of the current service. - if exe.user == "mz_system": - return exe.db.ports[ - "mz_system" if exe.mz_service == "materialized" else "mz_system2" - ] - return exe.db.ports[exe.mz_service] +class RevokeRoleAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "unknown role", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: with exe.db.lock: - if self.random_role and exe.db.roles: - user = self.rng.choice( - ["materialize", str(self.rng.choice(exe.db.roles))] + if len(exe.db.roles) < 2: + return False + role1, role2 = self.rng.sample(exe.db.roles, 2) + # Fixed lock order, as in GrantRoleAction: the sampled order + # deadlocks two workers that picked the same pair, and the REVOKE + # operands have to stay in the sampled order. + first, second = sorted((role1, role2), key=lambda role: role.role_id) + with first.lock, second.lock: + if role1 not in exe.db.roles or role2 not in exe.db.roles: + return False + exe.execute(f"REVOKE {role1} FROM {role2}", http=Http.RANDOM) + return True + + +class AlterOwnerAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = [ + "unknown role", + "must be a member of", + ] + super().errors_to_ignore(exe) + if exe.db.complexity in (Complexity.DDL, Complexity.DDLOnly): + result.extend(["does not exist"]) + return result + + def run(self, exe: Executor) -> bool: + # Only the snapshot needs the global lock, see CommentAction. + with exe.db.lock: + if not exe.db.roles: + return False + role = self.rng.choice(exe.db.roles) + tables = list(exe.db.tables) + views = list(exe.db.views) + direct_sources = exe.db.kafka_sources + exe.db.webhook_sources + table_sources = ( + exe.db.postgres_sources + + exe.db.mysql_sources + + exe.db.sql_server_sources + ) + loadgen_sources = list(exe.db.loadgen_sources) + sinks = exe.db.kafka_sinks + exe.db.iceberg_sinks + schemas = list(exe.db.schemas) + dbs = list(exe.db.dbs) + clusters = list(exe.db.clusters) + + candidates: list[tuple[str, str]] = [] + # Temp objects cannot change owner, they die with the session. + for table in tables: + if not table.temp: + candidates.append(("TABLE", str(table))) + for view in views: + if not view.temp: + candidates.append( + ( + "MATERIALIZED VIEW" if view.materialized else "VIEW", + str(view), + ) ) - else: - # Keep the executor's original user, e.g. the Cancel worker - # must stay mz_system or its cancels fail with "must be a - # member of" - user = exe.user - conn = exe.cur.connection + # Kafka and webhook source objects are readable directly, the + # others follow the source-table model: str(obj) names the table, + # the ingestion source is a separate catalog item. + for source in direct_sources: + candidates.append(("SOURCE", str(source))) + for source in table_sources: + candidates.append(("TABLE", str(source))) + candidates.append( + ( + "SOURCE", + f"{source.schema}.{identifier(source.executor.source)}", + ) + ) + for source in loadgen_sources: + candidates.append(("TABLE", str(source))) + candidates.append( + ("SOURCE", f"{source.schema}.{identifier(source.source_name())}") + ) + for sink in sinks: + candidates.append(("SINK", str(sink))) + for schema in schemas: + candidates.append(("SCHEMA", str(schema))) + for db in dbs: + candidates.append(("DATABASE", str(db))) + for cluster in clusters: + candidates.append(("CLUSTER", str(cluster))) + candidates.append(("SECRET", "materialize.public.pgpass")) + # NOTE: No CONNECTION target. Changing a connection's owner emits a + # Connection(Altered) implication, which re-alters every dependent + # sink's export connection. That re-alter can fail with InvalidAlter + # and panic the coordinator. + # See https://linear.app/materializeinc/issue/SQL-517 + kind, name = self.rng.choice(candidates) + with role.lock: + if role not in exe.db.roles: + return False + exe.execute(f"ALTER {kind} {name} OWNER TO {role}", http=Http.RANDOM) + return True - if exe.ws and exe.use_ws: - try: - exe.ws.close() - except: - pass - try: - exe.cur.close() - except: - pass - try: - conn.close() - except: - pass +class AlterDefaultPrivilegesAction(Action): + # Privileges per object type. The bool pair is (allows IN SCHEMA, allows IN + # DATABASE): schema-scoped objects accept both, SCHEMAS only IN DATABASE, + # DATABASES and CLUSTERS neither. Mixing e.g. ON DATABASES with IN DATABASE + # is a plan error, not a race, so it is generated out. + OBJECT_TYPES = { + "TABLES": (["SELECT", "INSERT", "UPDATE", "DELETE", "ALL"], True, True), + "TYPES": (["USAGE", "ALL"], True, True), + "SECRETS": (["USAGE", "ALL"], True, True), + "CONNECTIONS": (["USAGE", "ALL"], True, True), + "SCHEMAS": (["USAGE", "CREATE", "ALL"], False, True), + "DATABASES": (["USAGE", "CREATE", "ALL"], False, False), + "CLUSTERS": (["USAGE", "CREATE", "ALL"], False, False), + } - NUM_ATTEMPTS = 20 + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = [ + "unknown role", + "unknown schema", + "unknown database", + "must be a member of", + # FOR ALL ROLES and system-adjacent grants require privileges the + # (possibly reconnected-as-a-random-role) session may lack. + "permission denied to", + ] + super().errors_to_ignore(exe) + return result + + def run(self, exe: Executor) -> bool: + object_type = self.rng.choice(list(self.OBJECT_TYPES.keys())) + privileges, allows_in_schema, allows_in_database = self.OBJECT_TYPES[ + object_type + ] + privilege = self.rng.choice(privileges) + with exe.db.lock: + if not exe.db.roles: + return False + role = self.rng.choice(exe.db.roles) + for_clause = self.rng.choice( + ["FOR ALL ROLES"] + [f"FOR ROLE {r}" for r in exe.db.roles] + ) + in_clause = "" + if allows_in_schema and exe.db.schemas and self.rng.random() < 0.3: + in_clause = f" IN SCHEMA {self.rng.choice(exe.db.schemas)}" + elif allows_in_database and exe.db.dbs and self.rng.random() < 0.3: + in_clause = f" IN DATABASE {self.rng.choice(exe.db.dbs)}" + with role.lock: + if role not in exe.db.roles: + return False + if self.rng.choice([True, False]): + query = f"ALTER DEFAULT PRIVILEGES {for_clause}{in_clause} GRANT {privilege} ON {object_type} TO {role}" + else: + query = f"ALTER DEFAULT PRIVILEGES {for_clause}{in_clause} REVOKE {privilege} ON {object_type} FROM {role}" + exe.execute(query, http=Http.RANDOM) + return True + + +class BroadPrivilegesAction(Action): + """GRANT/REVOKE on object classes beyond the tables and views covered by + Grant/RevokePrivilegesAction.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "unknown role", + "unknown schema", + "unknown database", + "unknown cluster", + # System privileges require superuser, which a session reconnected + # as a random role does not have. + "permission denied to", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if not exe.db.roles: + return False + role = self.rng.choice(exe.db.roles) + targets: list[tuple[str, list[str]]] = [ + ("SYSTEM", ["CREATEDB", "CREATECLUSTER", "CREATEROLE", "ALL"]), + ("SECRET materialize.public.pgpass", ["USAGE", "ALL"]), + # NOTE: No CONNECTION target. GRANT/REVOKE on a connection emits + # a Connection(Altered) implication, which re-alters every + # dependent sink's export connection. That re-alter can fail + # with InvalidAlter and panic the coordinator. + # See https://linear.app/materializeinc/issue/SQL-517 + ] + if exe.db.schemas: + targets.append( + ( + f"SCHEMA {self.rng.choice(exe.db.schemas)}", + ["USAGE", "CREATE", "ALL"], + ) + ) + if exe.db.dbs: + targets.append( + ( + f"DATABASE {self.rng.choice(exe.db.dbs)}", + ["USAGE", "CREATE", "ALL"], + ) + ) + if exe.db.clusters: + targets.append( + ( + f"CLUSTER {self.rng.choice(exe.db.clusters)}", + ["USAGE", "CREATE", "ALL"], + ) + ) + target, privileges = self.rng.choice(targets) + privilege = self.rng.choice(privileges) + with role.lock: + if role not in exe.db.roles: + return False + if self.rng.choice([True, False]): + exe.execute( + f"GRANT {privilege} ON {target} TO {role}", http=Http.RANDOM + ) + else: + exe.execute( + f"REVOKE {privilege} ON {target} FROM {role}", http=Http.RANDOM + ) + return True + + +class ShowAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = super().errors_to_ignore(exe) + result.extend( + [ + # SHOW CREATE CLUSTER only works for managed clusters. We only + # target managed ones, this covers a managed->unmanaged race. + "SHOW CREATE for unmanaged clusters not yet supported", + # With auto_route_catalog_queries off, SHOW compiles to a + # catalog query on the active cluster, so it shares the read + # transaction's timedomain. + "in the same timedomain", + 'is not allowed from the "mz_catalog_server" cluster', + ] + ) + if exe.db.complexity in (Complexity.DDL, Complexity.DDLOnly): + result.extend(["does not exist"]) + return result + + def run(self, exe: Executor) -> bool: + if self.rng.choice([True, False]): + schema_scoped = [ + "SHOW TABLES", + "SHOW VIEWS", + "SHOW MATERIALIZED VIEWS", + "SHOW SOURCES", + "SHOW SINKS", + "SHOW INDEXES", + "SHOW OBJECTS", + "SHOW SECRETS", + "SHOW CONNECTIONS", + "SHOW TYPES", + ] + other = [ + "SHOW CLUSTERS", + "SHOW CLUSTER REPLICAS", + "SHOW DATABASES", + "SHOW SCHEMAS", + "SHOW ROLES", + "SHOW PRIVILEGES", + "SHOW DEFAULT PRIVILEGES", + "SHOW ROLE MEMBERSHIP", + "SHOW ALL", + ] + if self.rng.choice([True, False]): + query = self.rng.choice(schema_scoped) + with exe.db.lock: + if exe.db.schemas and self.rng.choice([True, False]): + query += f" FROM {self.rng.choice(exe.db.schemas)}" + if self.rng.random() < 0.2: + query += " LIKE '%1%'" + else: + query = self.rng.choice(other) + else: + # Only the snapshot needs the global lock, see CommentAction. + with exe.db.lock: + tables = list(exe.db.tables) + views = list(exe.db.views) + direct_sources = exe.db.kafka_sources + exe.db.webhook_sources + table_sources = ( + exe.db.postgres_sources + + exe.db.mysql_sources + + exe.db.sql_server_sources + ) + loadgen_sources = list(exe.db.loadgen_sources) + sinks = exe.db.kafka_sinks + exe.db.iceberg_sinks + indexes = list(exe.db.indexes) + clusters = list(exe.db.clusters) + + candidates: list[tuple[str, str]] = [ + ("CONNECTION", "materialize.public.kafka_conn"), + ("CONNECTION", "materialize.public.csr_conn"), + ] + for table in tables: + candidates.append(("TABLE", str(table))) + for view in views: + candidates.append( + ( + "MATERIALIZED VIEW" if view.materialized else "VIEW", + str(view), + ) + ) + # Kafka and webhook sources are readable directly. The others + # follow the source-table model where str() is the table and + # the ingestion source is a separate catalog item. + for source in direct_sources: + candidates.append(("SOURCE", str(source))) + for source in table_sources: + candidates.append(("TABLE", str(source))) + candidates.append( + ( + "SOURCE", + f"{source.schema}.{identifier(source.executor.source)}", + ) + ) + for source in loadgen_sources: + candidates.append(("TABLE", str(source))) + candidates.append( + ( + "SOURCE", + f"{source.schema}.{identifier(source.source_name())}", + ) + ) + for sink in sinks: + candidates.append(("SINK", str(sink))) + for index in indexes: + candidates.append(("INDEX", str(index))) + for cluster in clusters: + # SHOW CREATE CLUSTER is not supported for unmanaged + # clusters. + if cluster.managed: + candidates.append(("CLUSTER", str(cluster))) + kind, name = self.rng.choice(candidates) + # SHOW REDACTED CREATE CLUSTER is not supported + redacted = ( + "REDACTED " + if kind != "CLUSTER" and self.rng.choice([True, False]) + else "" + ) + query = f"SHOW {redacted}CREATE {kind} {name}" + exe.execute(query, http=Http.RANDOM, fetch=True) + return True + + +class SetSessionVariableAction(Action): + def __init__(self, rng: random.Random, composition: Composition | None): + super().__init__(rng, composition) + self.vars_with_values: dict[str, list[str]] = { + "statement_timeout": ["'30s'", "'60s'", "'0s'"], + "application_name": ["'parallel-workload'", "''"], + "client_min_messages": ["debug1", "info", "notice", "warning", "error"], + "max_query_result_size": ["100000", "1000000", "1000000000"], + "emit_timestamp_notice": ["true", "false"], + "emit_trace_id_notice": ["true", "false"], + # Only UTC is accepted, the rejection of other time zones is + # deliberate error path coverage. + "timezone": ["'UTC'", "'America/New_York'"], + } + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "invalid value for parameter", + "cannot have value", + "unrecognized configuration parameter", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + var = self.rng.choice(list(self.vars_with_values.keys())) + # statement_timeout is tracked on the executor, and only a statement on + # the worker's own session can change what that session times out at. + # Over the HTTP endpoint the SET or RESET lands on a one-shot session + # instead, leaving the tracking describing a value nothing has. + http = Http.NO if var == "statement_timeout" else Http.RANDOM + if self.rng.random() < 0.2: + exe.execute(f"RESET {var}", http=http) + if var == "statement_timeout": + # Back at the default timeout, so a timeout is a genuine + # finding again. + exe.statement_timeout_set = False + return True + value = self.rng.choice(self.vars_with_values[var]) + local = "LOCAL " if self.rng.random() < 0.1 else "" + exe.execute(f"SET {local}{var} = {value}", http=http) + if var == "statement_timeout": + # A timeout is expected for as long as this one is configured. A + # SET LOCAL keeps this set past the transaction that discards the + # value, which only errs towards tolerating more. + exe.statement_timeout_set = True + return True + + +class DiscardAction(Action): + def run(self, exe: Executor) -> bool: + # The session's temp objects die with DISCARD, drop them from the + # tracked state so other workers stop querying them, mirroring + # ReconnectAction. + if exe.temp_objects: + with exe.db.lock: + exe.db.tables[:] = [ + t for t in exe.db.tables if t not in exe.temp_objects + ] + exe.db.views[:] = [v for v in exe.db.views if v not in exe.temp_objects] + exe.temp_objects.clear() + # Only DISCARD TEMP, not DISCARD ALL. DISCARD ALL (like DEALLOCATE ALL) + # deallocates every prepared statement, including the ones psycopg + # transparently auto-prepares. psycopg's client-side cache would then + # be stale and the next reuse of such a statement fails with + # "prepared statement ... does not exist". DISCARD TEMP still exercises + # the temp-object teardown, which is the interesting path here. + exe.execute("DISCARD TEMP", http=Http.NO) + return True + + +class ValidateConnectionAction(Action): + # aws_conn is excluded, MinIO's STS support for validation is unclear. + CONNECTIONS = [ + "kafka_conn", + "csr_conn", + "postgres_conn", + "mysql_conn", + "sql_server_conn", + "polaris_conn", + ] + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "timeout: error trying to connect", + # A concurrent ALTER SECRET rotation (the workload only rotates a + # secret to its own value) can transiently expose an empty secret, + # so VALIDATE CONNECTION sends an empty password and the upstream + # system rejects it (secret-rotation atomicity). The same race + # surfaces with each upstream's own wording. + # TODO: Remove when https://linear.app/materializeinc/issue/SS-347 + # is fixed. + "empty password returned by client", # Postgres + "Access denied for user", # MySQL + "Login failed for user", # SQL Server + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + name = self.rng.choice(self.CONNECTIONS) + exe.execute(f"VALIDATE CONNECTION materialize.public.{name}", http=Http.NO) + return True + + +class AlterConnectionAction(Action): + # The SET clause per connection, setting the option to the value the + # connection already has. That still exercises the full reconfiguration + # path (restarting dependent sources and sinks) without breaking them. + # NOTE: BROKER takes no `=` (it is parsed specially), HOST/URL do. + SET_CLAUSES = { + "kafka_conn": "BROKER 'kafka:9092'", + "csr_conn": "URL = 'http://schema-registry:8081'", + "postgres_conn": "HOST = 'postgres'", + "mysql_conn": "HOST = 'mysql'", + "sql_server_conn": "HOST = 'sql-server'", + } + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "timeout: error trying to connect", + # The storage controller can refuse an in-place connection change + # depending on the connection's current state or dependents. + "cannot be altered in the requested way", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + name = self.rng.choice(list(self.SET_CLAUSES.keys())) + set_clause = self.SET_CLAUSES[name] + query = f"ALTER CONNECTION materialize.public.{name} SET ({set_clause})" + if self.rng.choice([True, False]): + validate = self.rng.choice(["true", "false"]) + query += f" WITH (VALIDATE = {validate})" + exe.execute(query, http=Http.RANDOM) + return True + + +class AlterSecretAction(Action): + def run(self, exe: Executor) -> bool: + # Rotate to the same value, exercising the rotation path (including + # dependent connections picking up the new secret version) without + # breaking the credentials. + name, value = self.rng.choice( + [ + ("pgpass", "postgres"), + ("mypass", MySql.DEFAULT_ROOT_PASSWORD), + ("sql_server_pass", SqlServer.DEFAULT_SA_PASSWORD), + ("minio", "minioadmin"), + ] + ) + exe.execute( + f"ALTER SECRET materialize.public.{name} AS '{value}'", http=Http.RANDOM + ) + return True + + +# TODO: Should factor this out so can easily use it without action +class ReconnectAction(Action): + def __init__( + self, + rng: random.Random, + composition: Composition | None, + random_role: bool = True, + ): + super().__init__(rng, composition) + self.random_role = random_role + + def run(self, exe: Executor) -> bool: + exe.mz_service = "materialized" + exe.log("reconnecting") + # The connection's temp objects die with it, drop them from the + # tracked state so other workers stop querying them. + if exe.temp_objects: + with exe.db.lock: + exe.db.tables[:] = [ + t for t in exe.db.tables if t not in exe.temp_objects + ] + exe.db.views[:] = [v for v in exe.db.views if v not in exe.temp_objects] + exe.temp_objects.clear() + host = exe.db.host + + def pg_port() -> int: + # System workers (e.g. the Cancel worker) live on the internal + # port, everyone else on the external one of the current service. + if exe.user == "mz_system": + return exe.db.ports[ + "mz_system" if exe.mz_service == "materialized" else "mz_system2" + ] + return exe.db.ports[exe.mz_service] + + with exe.db.lock: + if self.random_role and exe.db.roles: + user = self.rng.choice( + ["materialize", str(self.rng.choice(exe.db.roles))] + ) + else: + # Keep the executor's original user, e.g. the Cancel worker + # must stay mz_system or its cancels fail with "must be a + # member of" + user = exe.user + conn = exe.cur.connection + + if exe.ws and exe.use_ws: + try: + exe.ws.close() + except: + pass + + try: + exe.cur.close() + except: + pass + try: + conn.close() + except: + pass + + NUM_ATTEMPTS = 20 if exe.ws: for i in range( NUM_ATTEMPTS @@ -2996,6 +4623,10 @@ def pg_port() -> int: exe.set_isolation("SERIALIZABLE") cur.execute("SET auto_route_catalog_queries TO false") conn.autocommit = exe.autocommit + # A statement_timeout set on the old session died with it, so + # timeouts are genuine findings again unless a role default + # carries one over. + exe.statement_timeout_set = False try: cur.execute("SELECT pg_backend_pid()") except Exception as e: @@ -3070,7 +4701,11 @@ def __init__( def run(self, exe: Executor) -> bool: pid = self.rng.choice( - [worker.exe.pg_pid for worker in self.workers if worker.exe and worker.exe.pg_pid != -1] # type: ignore + [ + worker.exe.pg_pid + for worker in self.workers + if worker.exe and worker.exe.pg_pid != -1 + ] # type: ignore ) worker = None for i in range(len(self.workers)): @@ -3300,6 +4935,125 @@ def run(self, exe: Executor) -> bool: return True +class CreateLoadGeneratorSourceAction(Action): + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if len(exe.db.loadgen_sources) >= MAX_LOADGEN_SOURCES: + return False + source_id = exe.db.loadgen_source_id + exe.db.loadgen_source_id += 1 + try: + cluster = self.rng.choice(exe.db.clusters) + schema = self.rng.choice(exe.db.schemas) + except IndexError: + # We mostly prevent index errors, but we don't want to lock too + # much since that would reduce our chance of finding race + # conditions in production code, so ignore the rare case where + # we accidentally removed all objects. + return False + with schema.lock, cluster.lock: + if schema not in exe.db.schemas: + return False + if cluster not in exe.db.clusters: + return False + + source = LoadGeneratorSource(source_id, cluster, schema, self.rng) + source.create(exe) + exe.db.loadgen_sources.append(source) + return True + + +class DropLoadGeneratorSourceAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "still depended upon by", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if not exe.db.loadgen_sources: + return False + try: + source = self.rng.choice(exe.db.loadgen_sources) + except IndexError: + # We mostly prevent index errors, but we don't want to lock too + # much since that would reduce our chance of finding race + # conditions in production code, so ignore the rare case where + # we accidentally removed all objects. + return False + with source.lock: + # Was dropped while we were acquiring lock + if source not in exe.db.loadgen_sources: + return False + + exe.execute(f"DROP TABLE IF EXISTS {source}") + exe.execute( + f"DROP SOURCE {source.schema}.{identifier(source.source_name())} CASCADE", + http=Http.RANDOM, + ) + exe.db.loadgen_sources.remove(source) + return True + + +class CreateMultiLoadGeneratorSourceAction(Action): + GENERATORS = ["AUCTION", "TPCH", "MARKETING"] + + def errors_to_ignore(self, exe: Executor) -> list[str]: + # A rare cross-type subsource-name collision, or a concurrent create of + # the same generator type. + return ["already exists"] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + present = {s.generator for s in exe.db.multi_loadgen_sources} + available = [g for g in self.GENERATORS if g not in present] + if not available: + return False + generator = self.rng.choice(available) + source_id = exe.db.multi_loadgen_source_id + exe.db.multi_loadgen_source_id += 1 + try: + cluster = self.rng.choice(exe.db.clusters) + schema = self.rng.choice(exe.db.schemas) + except IndexError: + return False + with schema.lock, cluster.lock: + if schema not in exe.db.schemas: + return False + if cluster not in exe.db.clusters: + return False + source = MultiLoadGeneratorSource( + source_id, cluster, schema, generator, self.rng + ) + source.create(exe) + # NOTE: No db.lock around the append. Taking it here would nest it + # inside the schema and cluster locks, inverting the db.lock-first + # order every other action uses. list.append is atomic, and a + # concurrent same-generator create is caught server-side ("already + # exists", tolerated above). + exe.db.multi_loadgen_sources.append(source) + return True + + +class DropMultiLoadGeneratorSourceAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "still depended upon by", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if not exe.db.multi_loadgen_sources: + return False + source = self.rng.choice(exe.db.multi_loadgen_sources) + with source.lock: + if source not in exe.db.multi_loadgen_sources: + return False + exe.execute(f"DROP SOURCE {source} CASCADE", http=Http.RANDOM) + exe.db.multi_loadgen_sources.remove(source) + return True + + class CreateKafkaSourceAction(Action): def run(self, exe: Executor) -> bool: with exe.db.lock: @@ -3710,7 +5464,7 @@ def run(self, exe: Executor) -> bool: sink_id, cluster, schema, - self.rng.choice(exe.db.db_objects_without_views()), + self.rng.choice(exe.db.db_objects_for_sinks()), self.rng, ) sink.create(exe) @@ -3778,7 +5532,7 @@ def run(self, exe: Executor) -> bool: sink_id, cluster, schema, - self.rng.choice(exe.db.db_objects_without_views()), + self.rng.choice(exe.db.db_objects_for_sinks()), self.rng, ) sink.create(exe) @@ -3820,6 +5574,11 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: result = super().errors_to_ignore(exe) if exe.db.scenario == Scenario.Rename: result.extend(["404: no object was found at the path"]) + # DropSchemaCascadeAction / DropDatabaseCascadeAction can drop a + # webhook source concurrently without taking its per-object lock, so a + # POST that already picked the source can 404. + if exe.db.complexity in (Complexity.DDL, Complexity.DDLOnly): + result.extend(["404: no object was found at the path"]) return result def run(self, exe: Executor) -> bool: @@ -3884,6 +5643,383 @@ def run(self, exe: Executor) -> bool: return True +class AlterClusterSetAction(Action): + """Live reconfigure of a managed cluster (SIZE / REPLICATION FACTOR). + + Resizing or changing the replica count of a cluster hosting indexes, MVs, + sources, and sinks forces rehydration and replica teardown/spin-up under + concurrent DDL and DML.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + # A SET (SIZE) here or a ReconfigureCluster on the same cluster + # leaves a reconfiguration record in flight past the statement + # that started it. Replication factor is folded in at cut-over, + # so changing it meanwhile is refused. + "cannot change replication factor while a reconfiguration is in progress", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + # Cluster 0 stays fixed, it hosts sources and sinks. + managed = [c for c in exe.db.clusters[1:] if c.managed] + if not managed: + return False + cluster = self.rng.choice(managed) + with cluster.lock: + if cluster not in exe.db.clusters or not cluster.managed: + return False + choice = self.rng.choice(["size", "replication_factor", "reset_rf"]) + if choice == "size": + new_size = self.rng.choice(["scale=1,workers=1", "scale=1,workers=2"]) + exe.execute( + f"ALTER CLUSTER {cluster} SET (SIZE = '{new_size}')", + http=Http.RANDOM, + ) + cluster.size = new_size + for replica in cluster.replicas: + replica.size = new_size + elif choice == "replication_factor": + rf = self.rng.choice([1, 2]) + exe.execute( + f"ALTER CLUSTER {cluster} SET (REPLICATION FACTOR = {rf})", + http=Http.RANDOM, + ) + self._resize_replicas(cluster, rf) + else: + exe.execute( + f"ALTER CLUSTER {cluster} RESET (REPLICATION FACTOR)", + http=Http.RANDOM, + ) + self._resize_replicas(cluster, 1) + return True + + def _resize_replicas(self, cluster: Cluster, count: int) -> None: + # Managed cluster replicas are server-named (r1..rN), so the tracked + # list is rebuilt from scratch to match: the ids are what + # `ClusterReplica.name()` derives those names from. + cluster.replicas = [ + ClusterReplica(i, cluster.size, cluster) for i in range(count) + ] + cluster.replica_id = count + + +class DropSchemaCascadeAction(Action): + """DROP SCHEMA .. CASCADE, an atomic multi-object catalog mutation. + + Only enabled in DDL complexity: cross-schema dependents are cascade-dropped + server-side but stay tracked until they surface as "does not exist", which + DDL complexity ignores.""" + + def applicable(self, exe: Executor) -> bool: + return exe.db.complexity in (Complexity.DDL, Complexity.DDLOnly) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if len(exe.db.schemas) <= 1: + return False + schema = self.rng.choice(exe.db.schemas) + # Keep at least two non-temp tables alive outside the dropped + # schema. Query generation picks from the non-view objects, and a + # CASCADE that emptied that set would crash it. This mirrors + # DropTableAction's minimum. + if ( + len([t for t in exe.db.tables if not t.temp and t.schema is not schema]) + < 2 + ): + return False + with schema.lock: + if schema not in exe.db.schemas: + return False + if len(exe.db.schemas) <= 1: + return False + exe.execute(f"DROP SCHEMA {schema} CASCADE", http=Http.RANDOM) + exe.db.schemas.remove(schema) + untrack_objects_in_schemas(exe, {schema}) + return True + + +class DropDatabaseCascadeAction(Action): + """DROP DATABASE .. CASCADE, an atomic multi-object catalog mutation. + + DDL-complexity only, for the same reason as DropSchemaCascadeAction.""" + + def applicable(self, exe: Executor) -> bool: + return exe.db.complexity in (Complexity.DDL, Complexity.DDLOnly) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if len(exe.db.dbs) <= 1: + return False + db = self.rng.choice(exe.db.dbs) + # Keep at least two non-temp tables alive outside the dropped + # database, so query generation always has a non-view object. + if ( + len([t for t in exe.db.tables if not t.temp and t.schema.db is not db]) + < 2 + ): + return False + with db.lock: + if db not in exe.db.dbs: + return False + if len(exe.db.dbs) <= 1: + return False + exe.execute(f"DROP DATABASE {db} CASCADE", http=Http.RANDOM) + exe.db.dbs.remove(db) + with exe.db.lock: + dropped = {s for s in exe.db.schemas if s.db is db} + exe.db.schemas[:] = [s for s in exe.db.schemas if s.db is not db] + untrack_objects_in_schemas(exe, dropped) + return True + + +class CreateTypeAction(Action): + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if len(exe.db.types) >= MAX_TYPES: + return False + type_id = exe.db.type_id + exe.db.type_id += 1 + try: + schema = self.rng.choice(exe.db.schemas) + except IndexError: + return False + with schema.lock: + if schema not in exe.db.schemas: + return False + typ = Type(type_id, schema, self.rng) + typ.create(exe) + exe.db.types.append(typ) + return True + + +class DropTypeAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "still depended upon by", + "cannot be dropped", + # Another worker (or a CASCADE drop of the schema/database) can + # drop the type first. + "does not exist", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if not exe.db.types: + return False + typ = self.rng.choice(exe.db.types) + with typ.lock: + if typ not in exe.db.types: + return False + exe.execute(f"DROP TYPE {typ}", http=Http.RANDOM) + exe.db.types.remove(typ) + return True + + +class CreateNetworkPolicyAction(Action): + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if len(exe.db.network_policies) >= MAX_NETWORK_POLICIES: + return False + policy_id = exe.db.network_policy_id + exe.db.network_policy_id += 1 + policy = NetworkPolicy(policy_id, self.rng) + policy.create(exe) + with exe.db.lock: + exe.db.network_policies.append(policy) + return True + + +class AlterNetworkPolicyAction(Action): + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if not exe.db.network_policies: + return False + policy = self.rng.choice(exe.db.network_policies) + with policy.lock: + if policy not in exe.db.network_policies: + return False + exe.execute( + f"ALTER NETWORK POLICY {policy} SET ({policy.rules_clause()})", + http=Http.RANDOM, + ) + policy.num_rules = self.rng.randint(1, 3) + return True + + +class DropNetworkPolicyAction(Action): + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + # The policy is installed as a default somewhere (should not happen, + # we never install ours, but be safe). + "cannot be dropped", + # Another worker dropped the same policy first. The error carries + # the raw name ('netpol-N', not the quoted form), so DROP resolves + # the name correctly. This is a concurrency race, not the ALTER + # NETWORK POLICY quoted-name bug. + "unknown network policy", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + if not exe.db.network_policies: + return False + policy = self.rng.choice(exe.db.network_policies) + with policy.lock: + if policy not in exe.db.network_policies: + return False + exe.execute(f"DROP NETWORK POLICY {policy}", http=Http.RANDOM) + exe.db.network_policies.remove(policy) + return True + + +class SystemCatalogReadAction(Action): + """Read a random system-catalog / introspection relation while DDL churns. + + Exercises catalog-read-vs-write consistency and the auto-route path. + Relations that the `materialize` user cannot read (mz_notices, + mz_recent_activity_log_redacted) are left out.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + # An introspection view over many objects can outrun + # statement_timeout, never a real bug in a stress run. + "canceling statement due to statement timeout", + # Reading a system-catalog relation inside a read transaction that + # already touched user objects crosses timedomains. + "in the same timedomain", + 'is not allowed from the "mz_catalog_server" cluster', + ] + super().errors_to_ignore(exe) + + RELATIONS = [ + "mz_catalog.mz_objects", + "mz_catalog.mz_columns", + "mz_catalog.mz_indexes", + "mz_catalog.mz_sources", + "mz_catalog.mz_sinks", + "mz_catalog.mz_materialized_views", + "mz_catalog.mz_views", + "mz_catalog.mz_tables", + "mz_catalog.mz_audit_events", + "mz_catalog.mz_databases", + "mz_catalog.mz_schemas", + "mz_catalog.mz_roles", + "mz_catalog.mz_clusters", + "mz_catalog.mz_cluster_replicas", + "mz_internal.mz_frontiers", + "mz_internal.mz_hydration_statuses", + "mz_internal.mz_compute_dependencies", + "mz_internal.mz_source_statuses", + "mz_internal.mz_sink_statuses", + "mz_internal.mz_materialization_lag", + "mz_internal.mz_wallclock_global_lag_recent_history", + "mz_internal.mz_cluster_replica_statuses", + "mz_internal.mz_object_dependencies", + "mz_internal.mz_object_transitive_dependencies", + "mz_internal.mz_show_all_objects", + "mz_internal.mz_comments", + ] + + def run(self, exe: Executor) -> bool: + relation = self.rng.choice(self.RELATIONS) + exe.execute( + f"SELECT * FROM {relation} LIMIT {self.rng.randint(1, 100)}", + http=Http.RANDOM, + fetch=True, + ) + return True + + +class ExplainAnalyzeAction(Action): + """EXPLAIN ANALYZE against a live materialized view or index dataflow. + + Runs generated introspection queries on the active cluster. Racing it + against drop/replace of the target probes the introspection path.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + # The active cluster has more than one replica. + "log source reads must target a replica", + "does not exist", + "not been hydrated", + "not been materialized", + # A concurrent DROP/reconfigure of the targeted replica retires the + # introspection query. No panic in services.log, just a race. + "target replica failed or was dropped", + # Introspection over a large dataflow can outrun statement_timeout. + "canceling statement due to statement timeout", + # The generated introspection queries can cross timedomains or be + # routed to mz_catalog_server. + "in the same timedomain", + 'is not allowed from the "mz_catalog_server" cluster', + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + mvs = [v for v in exe.db.views if v.materialized] + with exe.db.lock: + indexes = list(exe.db.indexes) + candidates: list[tuple[str, DBObject | Index]] = [] + for v in mvs: + candidates.append(("MATERIALIZED VIEW", v)) + for i in indexes: + candidates.append(("INDEX", i)) + if not candidates: + return False + kind, obj = self.rng.choice(candidates) + analysis = self.rng.choice( + ["MEMORY", "CPU", "MEMORY WITH SKEW", "CPU WITH SKEW", "HINTS"] + ) + with obj.lock: + if kind == "MATERIALIZED VIEW" and obj not in exe.db.views: + return False + if kind == "INDEX" and obj not in exe.db.indexes: + return False + exe.execute( + f"EXPLAIN ANALYZE {analysis} FOR {kind} {obj}", + http=Http.NO, + fetch=True, + ) + return True + + +class ExplainFilterPushdownAction(Action): + """EXPLAIN FILTER PUSHDOWN, which inspects durable persist state to compute + which parts a query's filters would read.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + result = super().errors_to_ignore(exe) + result.extend( + [ + "in the same timedomain", + 'is not allowed from the "mz_catalog_server" cluster', + # Scanning persist part stats can outrun statement_timeout. + "canceling statement due to statement timeout", + ] + ) + if exe.db.complexity == Complexity.DDL: + result.extend(["does not exist"]) + return result + + def run(self, exe: Executor) -> bool: + mvs = [v for v in exe.db.views if v.materialized] + if mvs and self.rng.choice([True, False]): + view = self.rng.choice(mvs) + with view.lock: + if view not in exe.db.views: + return False + exe.execute( + f"EXPLAIN FILTER PUSHDOWN FOR MATERIALIZED VIEW {view}", + http=Http.NO, + fetch=True, + ) + else: + query = self.generate_select_query(exe, ExprKind.ALL) + exe.execute( + f"EXPLAIN FILTER PUSHDOWN FOR {query}", http=Http.NO, fetch=True + ) + return True + + class SourceSinkStallCheckAction(Action): def applicable(self, exe: Executor) -> bool: return exe.db.scenario not in ( @@ -3921,6 +6057,7 @@ def run(self, exe: Executor) -> bool: ("postgres_sources", exe.db.postgres_sources), ("mysql_sources", exe.db.mysql_sources), ("sql_server_sources", exe.db.sql_server_sources), + ("loadgen_sources", exe.db.loadgen_sources), ("webhook_sources", exe.db.webhook_sources), ]: counts = [] @@ -3932,6 +6069,85 @@ def run(self, exe: Executor) -> bool: return True +class DependencyConsistencyAction(Action): + """Client-side catalog dependency oracle: flag any dependency edge whose + endpoints are not both live objects (a dangling uses/used_by edge). + + PREPARED BUT DISABLED (commented out of read_action_list). This targets the + SQL-521 class: a sink left pointing at a materialized view that was dropped + under cancel, i.e. the same MissingUses inconsistency the coordinator's own + check_consistency asserts on. While SQL-521 is open the workload + legitimately produces such danglers, so enabling this now would re-detect + the known corruption rather than find new bugs. + TODO: enable in read_action_list once SQL-521 is fixed. Verify the + mz_internal.mz_object_dependencies column names (object_id, + referenced_object_id) still hold when enabling.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + # Reading a catalog relation inside a read txn that already touched + # user objects crosses timedomains. + "in the same timedomain", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + exe.execute( + "SELECT d.object_id, d.referenced_object_id " + "FROM mz_internal.mz_object_dependencies d " + "LEFT JOIN mz_objects o1 ON d.object_id = o1.id " + "LEFT JOIN mz_objects o2 ON d.referenced_object_id = o2.id " + "WHERE o1.id IS NULL OR o2.id IS NULL", + http=Http.NO, + ) + dangling = exe.cur.fetchall() + if dangling: + raise ValueError( + f"dangling catalog dependency edges (SQL-521 class): {dangling[:5]}" + ) + return True + + +class SourceReadHoldSweepAction(Action): + """Read a source-backed relation to force read-hold acquisition on the + source's remap shard. + + PREPARED BUT DISABLED (commented out of read_action_list). Intended for the + kill scenario, where racing envd/clusterd restarts stresses read-hold + reinstatement: the SS-346 class (a dependent's read hold on a source's remap + shard not upheld across a restart, so its since advances past the + dependent's upper) and PER-49 (a compute import as_of behind the compacted + since after ALTER TABLE ADD COLUMN + kill). Enabling it now just re-triggers + those known coordinator/compute panics. + TODO: enable in read_action_list once SS-346 and PER-49 are fixed.""" + + def errors_to_ignore(self, exe: Executor) -> list[str]: + return [ + "in the same timedomain", + ] + super().errors_to_ignore(exe) + + def run(self, exe: Executor) -> bool: + with exe.db.lock: + sources = [ + o + for o in exe.db.db_objects() + if isinstance( + o, + LoadGeneratorSource + | KafkaSource + | PostgresSource + | MySqlSource + | SqlServerSource + | WebhookSource, + ) + ] + if not sources: + return False + obj = self.rng.choice(sources) + exe.execute(f"SELECT count(*) FROM {obj}", http=Http.RANDOM) + exe.cur.fetchall() + return True + + class ActionList: action_classes: list[type[Action]] weights: list[float] @@ -3949,15 +6165,31 @@ def __init__( [ (SelectAction, 100), (SelectOneAction, 1), + (ParameterizedQueryAction, 20), # (SQLsmithAction, 30), # Questionable use ( CopyToS3Action, 100, ), + (CopyToStdoutAction, 20), + (ShowAction, 10), + (SystemCatalogReadAction, 10), + # TODO: Reenable once EXPLAIN FILTER PUSHDOWN can no longer panic the + # coordinator when a referenced compute collection is concurrently + # dropped. sequence_explain_pushdown -> acquire_read_holds().expect( + # "missing compute collection") at read_policy.rs:389 (normal peeks and + # EXPLAIN ANALYZE handle the drop gracefully). + # See https://linear.app/materializeinc/issue/SQL-519 + # (ExplainFilterPushdownAction, 5), + # PREPARED BUT DISABLED (see class docstrings): enabling these now just + # re-detects known-unfixed coordinator bugs rather than finding new ones. + # (DependencyConsistencyAction, 5), # TODO: enable once SQL-521 fixed + # (SourceReadHoldSweepAction, 5), # TODO: enable once SS-346 & PER-49 fixed (SetClusterAction, 1), (CommitRollbackAction, 30), (ReconnectAction, 1), (FlipFlagsAction, 2), + (SetSessionVariableAction, 2), ], autocommit=False, ) @@ -4003,6 +6235,8 @@ def __init__( (SetClusterAction, 1), (ReconnectAction, 1), (FlipFlagsAction, 2), + (SetSessionVariableAction, 2), + (DiscardAction, 2), # TODO: Reenable when SS-193 and SS-325 are fixed # (SourceSinkStallCheckAction, 4), # (TransactionIsolationAction, 1), @@ -4018,10 +6252,13 @@ def __init__( (DropTableAction, 2), (CreateViewAction, 8), (DropViewAction, 8), + (CreateOrReplaceViewAction, 4), (CreateRoleAction, 2), (DropRoleAction, 2), + (AlterRoleAction, 2), (CreateClusterAction, 1), (DropClusterAction, 1), + (AlterClusterSetAction, 3), (SwapClusterAction, 10), (CreateClusterReplicaAction, 2), (DropClusterReplicaAction, 2), @@ -4035,8 +6272,13 @@ def __init__( (DropIcebergSinkAction, 4), (CreateKafkaSourceAction, 4), (DropKafkaSourceAction, 4), - (CreateMySqlSourceAction, 4), - (DropMySqlSourceAction, 4), + (CreateLoadGeneratorSourceAction, 4), + (DropLoadGeneratorSourceAction, 4), + (CreateMultiLoadGeneratorSourceAction, 2), + (DropMultiLoadGeneratorSourceAction, 2), + # TODO: Reenable when https://linear.app/materializeinc/issue/SS-307 is fixed + # (CreateMySqlSourceAction, 4), + # (DropMySqlSourceAction, 4), (CreatePostgresSourceAction, 4), (DropPostgresSourceAction, 4), # TODO: Reenable when https://linear.app/materializeinc/issue/SS-290 is fixed @@ -4044,18 +6286,64 @@ def __init__( # (DropSqlServerSourceAction, 4), (GrantPrivilegesAction, 4), (RevokePrivilegesAction, 1), + (GrantRoleAction, 2), + (RevokeRoleAction, 1), + (AlterOwnerAction, 2), + (AlterDefaultPrivilegesAction, 2), + (BroadPrivilegesAction, 2), + (ShowAction, 4), + (ValidateConnectionAction, 2), + # TODO: Reenable once altering a connection that sinks or sources depend + # on can no longer panic the coordinator. Re-altering a dependent sink's + # export connection after the txn fails with InvalidAlter, which + # unwrap_or_terminate turns into a panic. + # See https://linear.app/materializeinc/issue/SQL-517 + # (AlterConnectionAction, 2), + (AlterSecretAction, 2), (ReconnectAction, 1), (CreateDatabaseAction, 1), (DropDatabaseAction, 1), + # TODO: Reenable once a concurrent DROP DATABASE CASCADE can no longer + # panic the coordinator. A staged create (e.g. the source executor's + # CREATE SECRET) whose target database is dropped between staging and + # finish hits resolve_full_name -> get_database (panicking OrdMap index) + # in catalog transact_op. Only CASCADE can drop a non-empty database, + # so this is the precise trigger. + # See https://linear.app/materializeinc/issue/SQL-518 + # (DropDatabaseCascadeAction, 1), (CreateSchemaAction, 1), (DropSchemaAction, 1), + (DropSchemaCascadeAction, 1), + (CreateTypeAction, 2), + (DropTypeAction, 2), + (CreateNetworkPolicyAction, 1), + # TODO: Reenable once ALTER NETWORK POLICY resolves quoted (e.g. + # hyphenated) names. It looks the policy up by its quoted display form, + # so it fails with "unknown network policy" for any name that requires + # quoting, even though CREATE and DROP work. + # See https://linear.app/materializeinc/issue/CLO-143 + # (AlterNetworkPolicyAction, 1), + (DropNetworkPolicyAction, 1), (RenameSchemaAction, 10), (RenameTableAction, 10), (RenameViewAction, 10), (RenameKafkaSinkAction, 10), (RenameIcebergSinkAction, 10), (SwapSchemaAction, 10), - (ReplaceMaterializedViewAction, 20), + (CreateReplacementMaterializedViewAction, 10), + (ApplyReplacementMaterializedViewAction, 5), + (DropReplacementMaterializedViewAction, 5), + (SealedCollectionCheckAction, 2), + (TransactionIsolationAction, 1), + (BoundedStalenessReadAction, 2), + (ReadOnlyTransactionAction, 3), + (DDLTransactionAction, 2), + (SystemCatalogReadAction, 4), + (ExplainAnalyzeAction, 4), + # TODO: Reenable with EXPLAIN FILTER PUSHDOWN's coordinator panic on a + # concurrently-dropped compute collection (read_policy.rs:389). + # See https://linear.app/materializeinc/issue/SQL-519 + # (ExplainFilterPushdownAction, 2), (FlipFlagsAction, 2), # TODO: Reenable when https://linear.app/materializeinc/issue/SQL-405 is fixed. # (AlterTableAddColumnAction, 10), diff --git a/misc/python/materialize/parallel_workload/column.py b/misc/python/materialize/parallel_workload/column.py index 9a874d71443c6..9ec25e15de454 100644 --- a/misc/python/materialize/parallel_workload/column.py +++ b/misc/python/materialize/parallel_workload/column.py @@ -88,7 +88,11 @@ def create(self) -> str: return result -class WebhookColumn(Column): +class SourceColumn(Column): + """Column of a source, taking its name from the upstream schema instead of + generating one. The per-source subclasses only exist to keep the column + lists of the different source kinds distinct in type annotations.""" + def __init__( self, name: str, @@ -105,69 +109,25 @@ def name(self, in_query: bool = False) -> str: return identifier(self.raw_name) if in_query else self.raw_name -class KafkaColumn(Column): - def __init__( - self, - name: str, - data_type: type[DataType], - nullable: bool, - db_object: "DBObject", - ): - self.raw_name = name - self.data_type = data_type - self.nullable = nullable - self.db_object = db_object +class WebhookColumn(SourceColumn): + pass - def name(self, in_query: bool = False) -> str: - return identifier(self.raw_name) if in_query else self.raw_name +class KafkaColumn(SourceColumn): + pass -class MySqlColumn(Column): - def __init__( - self, - name: str, - data_type: type[DataType], - nullable: bool, - db_object: "DBObject", - ): - self.raw_name = name - self.data_type = data_type - self.nullable = nullable - self.db_object = db_object - def name(self, in_query: bool = False) -> str: - return identifier(self.raw_name) if in_query else self.raw_name +class LoadGeneratorColumn(SourceColumn): + pass -class PostgresColumn(Column): - def __init__( - self, - name: str, - data_type: type[DataType], - nullable: bool, - db_object: "DBObject", - ): - self.raw_name = name - self.data_type = data_type - self.nullable = nullable - self.db_object = db_object +class MySqlColumn(SourceColumn): + pass - def name(self, in_query: bool = False) -> str: - return identifier(self.raw_name) if in_query else self.raw_name +class PostgresColumn(SourceColumn): + pass -class SqlServerColumn(Column): - def __init__( - self, - name: str, - data_type: type[DataType], - nullable: bool, - db_object: "DBObject", - ): - self.raw_name = name - self.data_type = data_type - self.nullable = nullable - self.db_object = db_object - def name(self, in_query: bool = False) -> str: - return identifier(self.raw_name) if in_query else self.raw_name +class SqlServerColumn(SourceColumn): + pass diff --git a/misc/python/materialize/parallel_workload/database.py b/misc/python/materialize/parallel_workload/database.py index 8920be86fbad6..b3d72d2fbb281 100644 --- a/misc/python/materialize/parallel_workload/database.py +++ b/misc/python/materialize/parallel_workload/database.py @@ -47,6 +47,7 @@ from materialize.parallel_workload.column import ( Column, KafkaColumn, + LoadGeneratorColumn, MySqlColumn, PostgresColumn, SqlServerColumn, @@ -60,7 +61,18 @@ MAX_COLUMNS = 50 MAX_INCLUDE_HEADERS = 5 -MAX_ROWS = 500 +# The row count a view's join squares. A view over two tables joins them on a +# random boolean predicate, so the join is a cross product and every read of +# that view builds MAX_ROWS**2 wide intermediate rows, whatever LIMIT the +# reading query carries. Measured at 500 with only four columns, one +# `COPY (SELECT * FROM view WHERE .. LIMIT 100) TO 's3://..'` cost ~450 MiB of +# replica RSS, and pw tables carry up to MAX_COLUMNS of them. A handful of such +# reads at once is what grew a clusterd to 19 GiB and had the kernel OOM-killer +# take it, plus environmentd and the Kafka broker, out of the one cgroup all of +# a run's containers share (nightlies 17660-17701). 100 keeps enough rows for +# the DML, persist and index paths to be interesting while cutting the +# intermediate 25-fold. +MAX_ROWS = 100 MAX_CLUSTERS = 4 MAX_CLUSTER_REPLICAS = 2 MAX_DBS = 50 @@ -74,8 +86,11 @@ MAX_MYSQL_SOURCES = 50 MAX_SQL_SERVER_SOURCES = 50 MAX_POSTGRES_SOURCES = 50 +MAX_LOADGEN_SOURCES = 50 MAX_KAFKA_SINKS = 50 MAX_ICEBERG_SINKS = 50 +MAX_TYPES = 50 +MAX_NETWORK_POLICIES = 30 MAX_INITIAL_DBS = 1 MAX_INITIAL_SCHEMAS = 1 @@ -88,6 +103,7 @@ MAX_INITIAL_MYSQL_SOURCES = 1 MAX_INITIAL_SQL_SERVER_SOURCES = 1 MAX_INITIAL_POSTGRES_SOURCES = 1 +MAX_INITIAL_LOADGEN_SOURCES = 1 class BodyFormat(Enum): @@ -165,6 +181,13 @@ def name(self) -> str: class DBObject: columns: list[Column] lock: threading.Lock + # Whether reading from this object can legitimately reach the empty + # (sealed) frontier: its own shard seals in normal operation, or a + # dataflow reading it sees an input frontier that becomes empty. Plain + # tables and unbounded sources never seal, so the default is False. + # Bounded (UP TO) load generators seal once they finish, and views + # propagate sealing from their inputs, materialized or not. + can_seal: bool = False def __init__(self): self.lock = threading.Lock() @@ -283,8 +306,9 @@ def __init__( rng.choice( [ "ON COMMIT", - f"EVERY '{rng.randint(1, 60)} seconds {rng.randint(0, 60)} minutes'", - f"EVERY '{rng.randint(1, 60)} seconds {rng.randint(0, 60)} minutes' ALIGNED TO (mz_now())", + # TODO: Restore minute-scale intervals when CPU-196 is fixed + f"EVERY '{rng.randint(1, 15)} seconds'", + f"EVERY '{rng.randint(1, 15)} seconds' ALIGNED TO (mz_now())", # Always in the future of all refreshes of previously generated MVs "AT mz_now()::string::int8 + 1000", ] @@ -293,6 +317,21 @@ def __init__( else None ) + # A materialized view's shard seals (its write frontier advances to the + # empty frontier) once it can never produce more output. That happens + # for REFRESH AT views after their last refresh, for repeat_row(-1) + # constant views on hydration, and transitively for any view that reads + # from an input that itself seals. The replacement and sealed-shard + # oracles key off this to tell legitimate seals from wrongly finalized + # shards. Unmaterialized views have no shard, but a dataflow reading + # one inlines its inputs, so sealing must propagate through them too. + self.can_seal = ( + (self.refresh or "").startswith("AT") + or self.repeat_row_const + or base_object.can_seal + or (base_object2 is not None and base_object2.can_seal) + ) + if base_object2: self.on_expr = expression( Boolean, all_columns, rng, kind=ExprKind.MATERIALIZABLE @@ -370,8 +409,12 @@ def select_str(exprs: str) -> str: return query - def create(self, exe: Executor) -> None: + def create(self, exe: Executor, or_replace: bool = False) -> None: query = "CREATE " + # OR REPLACE keeps the same catalog item, swapping its definition and + # rebuilding the dataflow. It is incompatible with TEMP. + if or_replace and not self.temp: + query += "OR REPLACE " if self.temp: query += "TEMP " if self.materialized: @@ -576,14 +619,19 @@ def __init__( self.cluster = cluster self.schema = schema self.base_object = base_object - key_cols = [ - column - for column in rng.sample( - base_object.columns, k=rng.randint(1, len(base_object.columns)) - ) - ] - key_col_names = [column.name(True) for column in key_cols] - self.key = f"KEY ({', '.join(key_col_names)}) NOT ENFORCED" + self.mode = rng.choice(["UPSERT", "APPEND"]) + if self.mode == "UPSERT": + key_cols = [ + column + for column in rng.sample( + base_object.columns, k=rng.randint(1, len(base_object.columns)) + ) + ] + key_col_names = [column.name(True) for column in key_cols] + self.key = f"KEY ({', '.join(key_col_names)}) NOT ENFORCED" + else: + # APPEND mode does not permit a KEY. + self.key = "" self.table_name = f"icesink_topic{self.sink_id}_{uuid.uuid4().hex[:8]}" self.rename = 0 @@ -596,7 +644,7 @@ def __str__(self) -> str: return f"{self.schema}.{identifier(self.name())}" def create(self, exe: Executor) -> None: - query = f"CREATE SINK {self} IN CLUSTER {self.cluster} FROM {self.base_object} INTO ICEBERG CATALOG CONNECTION polaris_conn (NAMESPACE 'default_namespace', TABLE '{self.table_name}') USING AWS CONNECTION aws_conn {self.key} MODE UPSERT WITH (COMMIT INTERVAL '1s')" + query = f"CREATE SINK {self} IN CLUSTER {self.cluster} FROM {self.base_object} INTO ICEBERG CATALOG CONNECTION polaris_conn (NAMESPACE 'default_namespace', TABLE '{self.table_name}') USING AWS CONNECTION aws_conn {self.key} MODE {self.mode} WITH (COMMIT INTERVAL '1s')" exe.execute(query) @@ -608,6 +656,8 @@ class KafkaSink(DBObject): base_object: DBObject envelope: str key: str + connection_options: list[str] + no_snapshot: bool def __init__( self, @@ -622,6 +672,19 @@ def __init__( self.cluster = cluster self.schema = schema self.base_object = base_object + self.connection_options = [] + if rng.random() < 0.3: + compression = rng.choice(["none", "gzip", "lz4", "zstd", "snappy"]) + self.connection_options.append(f"COMPRESSION TYPE = '{compression}'") + if rng.random() < 0.2: + self.connection_options.append( + f"TRANSACTIONAL ID PREFIX 'pw-txn-{self.sink_id}'" + ) + if rng.random() < 0.2: + self.connection_options.append( + f"PROGRESS GROUP ID PREFIX 'pw-progress-{self.sink_id}'" + ) + self.no_snapshot = rng.random() < 0.2 universal_formats = [ "FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn", "FORMAT JSON", @@ -681,7 +744,10 @@ def create(self, exe: Executor) -> None: if self.partition_count else "" ) - query = f"CREATE SINK {self} IN CLUSTER {self.cluster} FROM {self.base_object} INTO KAFKA CONNECTION kafka_conn (TOPIC {topic}{maybe_partition}) {self.key} {self.format} ENVELOPE {self.envelope}" + options = "".join(f", {option}" for option in self.connection_options) + query = f"CREATE SINK {self} IN CLUSTER {self.cluster} FROM {self.base_object} INTO KAFKA CONNECTION kafka_conn (TOPIC {topic}{maybe_partition}{options}) {self.key} {self.format} ENVELOPE {self.envelope}" + if self.no_snapshot: + query += " WITH (SNAPSHOT = false)" exe.execute(query) @@ -861,6 +927,110 @@ def create(self, exe: Executor) -> None: self.executor.create(logging_exe=exe) +class LoadGeneratorSource(DBObject): + """A COUNTER load generator source. Always bounded by UP TO so it cannot + grow without limit, which also exercises the finished-source lifecycle + state. The readable object (str(self)) is the table created from the + source, matching the source-table model the other sources use.""" + + # A finished (UP TO reached) counter advances the source table's frontier + # to the empty antichain, so anything reading it seals legitimately. + can_seal = True + + source_id: int + cluster: "Cluster" + schema: Schema + columns: list[LoadGeneratorColumn] + tick_interval: str + up_to: int + + def __init__( + self, + source_id: int, + cluster: "Cluster", + schema: Schema, + rng: random.Random, + ): + super().__init__() + self.source_id = source_id + self.cluster = cluster + self.schema = schema + self.tick_interval = rng.choice(["10ms", "100ms", "1s"]) + # Bounded near MAX_ROWS: this source-table is a readable relation that + # views/MVs join over, and unlike user tables it is not capped by + # MAX_ROWS. A large counter feeding a maintained join is a clusterd + # memory risk, so keep it in the same size class as the other data. + self.up_to = rng.randint(1, MAX_ROWS) + self.columns = [LoadGeneratorColumn("counter", Long, False, self)] + + def name(self) -> str: + return naughtify(f"lg-{self.source_id}") + + def source_name(self) -> str: + return naughtify(f"lg-src-{self.source_id}") + + def __str__(self) -> str: + return f"{self.schema}.{identifier(self.name())}" + + def create(self, exe: Executor) -> None: + source = f"{self.schema}.{identifier(self.source_name())}" + exe.execute( + f"CREATE SOURCE {source} IN CLUSTER {self.cluster} FROM LOAD GENERATOR COUNTER (TICK INTERVAL '{self.tick_interval}', UP TO {self.up_to})" + ) + exe.execute(f"CREATE TABLE {self} FROM SOURCE {source}") + + +class MultiLoadGeneratorSource: + """A multi-subsource load generator (AUCTION / TPCH / MARKETING) created + with FOR ALL TABLES. Tracked create/drop-only, not as a readable relation: + its subsources have fixed names and hardcoded schemas, so wiring them into + the general read/expression machinery is out of scope. The value is the + multi-subsource source lifecycle and CASCADE teardown under concurrency. + + At most one source per generator type exists at a time, because FOR ALL + TABLES names its subsources fixedly (a second AUCTION would collide on + `auctions`, `bids`, ...).""" + + source_id: int + cluster: "Cluster" + schema: Schema + generator: str + tick_interval: str + lock: threading.Lock + + def __init__( + self, + source_id: int, + cluster: "Cluster", + schema: Schema, + generator: str, + rng: random.Random, + ): + self.source_id = source_id + self.cluster = cluster + self.schema = schema + self.generator = generator + self.tick_interval = rng.choice(["100ms", "1s"]) + self.lock = threading.Lock() + + def name(self) -> str: + return naughtify(f"mlg-{self.source_id}") + + def __str__(self) -> str: + return f"{self.schema}.{identifier(self.name())}" + + def create(self, exe: Executor) -> None: + options = [f"TICK INTERVAL '{self.tick_interval}'"] + if self.generator == "TPCH": + options.append("SCALE FACTOR 0.0001") + query = ( + f"CREATE SOURCE {self} IN CLUSTER {self.cluster} " + f"FROM LOAD GENERATOR {self.generator} ({', '.join(options)}) " + f"FOR ALL TABLES" + ) + exe.execute(query) + + class S3Object(DBObject): """A COPY TO dump of `table` that CopyFromS3Action can load back into it. @@ -1066,6 +1236,11 @@ def __str__(self) -> str: def create(self, exe: Executor) -> None: exe.execute(f"CREATE ROLE {self}") + # Make the workload's own user (materialize) a member, so it can + # manage the role, e.g. transfer object ownership to it: ALTER .. + # OWNER TO requires the executor to be a member of . + # The creating session has admin on the role it just created. + exe.execute(f"GRANT {self} TO materialize") class ClusterReplica: @@ -1083,6 +1258,12 @@ def __init__(self, replica_id: int, size: str, cluster: "Cluster"): self.lock = threading.Lock() def name(self) -> str: + # A managed cluster's replicas are named by the controller as r1..rN, + # never by us, so neither the rename counter nor naughtify applies. + # Rendering our own name there would make every reference to the + # replica, e.g. a replica-targeted materialized view, fail to resolve. + if self.cluster.managed: + return f"r{self.replica_id+1}" if self.rename: return naughtify(f"r-{self.replica_id+1}-{self.rename}") return naughtify(f"r-{self.replica_id+1}") @@ -1147,6 +1328,78 @@ def create(self, exe: Executor) -> None: exe.execute(query) +class Type: + """A user-defined type (row, list, or map). Schema-scoped.""" + + type_id: int + schema: Schema + kind: str + lock: threading.Lock + rng: random.Random + + def __init__(self, type_id: int, schema: Schema, rng: random.Random): + self.type_id = type_id + self.schema = schema + self.kind = rng.choice(["row", "list", "map"]) + self.rng = rng + self.lock = threading.Lock() + + def name(self) -> str: + return naughtify(f"type-{self.type_id}") + + def __str__(self) -> str: + return f"{self.schema}.{identifier(self.name())}" + + def create(self, exe: Executor) -> None: + if self.kind == "row": + scalar_types = ["int4", "text", "bool", "float8", "timestamp"] + fields = ", ".join( + f"f{i} {self.rng.choice(scalar_types)}" + for i in range(self.rng.randint(1, 4)) + ) + query = f"CREATE TYPE {self} AS ({fields})" + elif self.kind == "list": + element = self.rng.choice(["int4", "text", "float8"]) + query = f"CREATE TYPE {self} AS LIST (ELEMENT TYPE = {element})" + else: + value = self.rng.choice(["int4", "text", "float8"]) + query = f"CREATE TYPE {self} AS MAP (KEY TYPE = text, VALUE TYPE = {value})" + exe.execute(query) + + +class NetworkPolicy: + """A network policy. Top-level (not schema-scoped), like clusters. + + Rules are always allow-all so the workload can never lock itself out of + its own connections. The policy is never installed as the active + `network_policy` system parameter for the same reason.""" + + policy_id: int + num_rules: int + lock: threading.Lock + + def __init__(self, policy_id: int, rng: random.Random): + self.policy_id = policy_id + self.num_rules = rng.randint(1, 3) + self.lock = threading.Lock() + + def name(self) -> str: + return naughtify(f"netpol-{self.policy_id}") + + def __str__(self) -> str: + return identifier(self.name()) + + def rules_clause(self) -> str: + rules = ", ".join( + f"r{i} (action='allow', direction='ingress', address='0.0.0.0/0')" + for i in range(self.num_rules) + ) + return f"RULES ({rules})" + + def create(self, exe: Executor) -> None: + exe.execute(f"CREATE NETWORK POLICY {self} ({self.rules_clause()})") + + # TODO: Can access both databases from same connection! class Database: complexity: Complexity @@ -1176,10 +1429,18 @@ class Database: postgres_source_id: int sql_server_sources: list[SqlServerSource] sql_server_source_id: int + loadgen_sources: list[LoadGeneratorSource] + loadgen_source_id: int + multi_loadgen_sources: list[MultiLoadGeneratorSource] + multi_loadgen_source_id: int iceberg_sinks: list[IcebergSink] iceberg_sink_id: int kafka_sinks: list[KafkaSink] kafka_sink_id: int + types: list[Type] + type_id: int + network_policies: list[NetworkPolicy] + network_policy_id: int s3_path: int s3_objects: list[S3Object] read_then_write_counter: ReadThenWriteCounter @@ -1243,9 +1504,7 @@ def __init__( Cluster( i, managed=rng.choice([True, False]), - size=rng.choice( - ["scale=1,workers=1", "scale=1,workers=4", "scale=2,workers=2"] - ), + size=rng.choice(["scale=1,workers=1", "scale=1,workers=2"]), replication_factor=1, introspection_interval="1s", ) @@ -1262,6 +1521,12 @@ def __init__( self.mysql_sources = [] self.postgres_sources = [] self.sql_server_sources = [] + self.loadgen_sources = [ + LoadGeneratorSource( + i, rng.choice(self.clusters), rng.choice(self.schemas), rng + ) + for i in range(rng.randint(0, MAX_INITIAL_LOADGEN_SOURCES)) + ] self.iceberg_sinks = [] self.kafka_sinks = [] self.s3_objects = [] @@ -1269,9 +1534,16 @@ def __init__( self.mysql_source_id = len(self.mysql_sources) self.postgres_source_id = len(self.postgres_sources) self.sql_server_source_id = len(self.sql_server_sources) + self.loadgen_source_id = len(self.loadgen_sources) + self.multi_loadgen_sources = [] + self.multi_loadgen_source_id = 0 self.iceberg_sink_id = len(self.iceberg_sinks) self.kafka_sink_id = len(self.kafka_sinks) self.read_then_write_counter = ReadThenWriteCounter() + self.types = [] + self.type_id = 0 + self.network_policies = [] + self.network_policy_id = 0 self.lock = threading.Lock() self.sqlsmith_state = "" self.flags = {} @@ -1284,6 +1556,7 @@ def db_objects( | PostgresSource | SqlServerSource | KafkaSource + | LoadGeneratorSource | View | Table ]: @@ -1300,6 +1573,7 @@ def db_objects( + self.mysql_sources + self.postgres_sources + self.sql_server_sources + + self.loadgen_sources + self.webhook_sources ) @@ -1311,6 +1585,7 @@ def db_objects_without_views( | PostgresSource | SqlServerSource | KafkaSource + | LoadGeneratorSource | View | Table ]: @@ -1318,6 +1593,28 @@ def db_objects_without_views( obj for obj in self.db_objects() if type(obj) != View or obj.materialized ] + def db_objects_for_sinks( + self, + ) -> list[ + WebhookSource + | MySqlSource + | PostgresSource + | SqlServerSource + | KafkaSource + | View + | Table + ]: + """Objects usable as a sink's input (base object or ALTER SINK SET + FROM target). Load generator source tables are excluded: an + ALTER SINK .. SET FROM one can trigger the sink stall of + https://linear.app/materializeinc/issue/SS-344, which is worse for a + continuously-producing input.""" + return [ + obj + for obj in self.db_objects_without_views() + if not isinstance(obj, LoadGeneratorSource) + ] + def __iter__(self): """Returns all relations""" return ( @@ -1342,6 +1639,13 @@ def create(self, exe: Executor, composition: Composition) -> None: for row in exe.cur.fetchall(): exe.execute(f"DROP ROLE {identifier(row[0])}") + # Network policies survive restarts and are top-level, so leftovers + # from a killed run have to be swept before recreating. + exe.execute("SELECT name FROM mz_internal.mz_network_policies") + for row in exe.cur.fetchall(): + if row[0].startswith("netpol-"): + exe.execute(f"DROP NETWORK POLICY {identifier(row[0])}") + print("Creating connections") exe.execute( diff --git a/misc/python/materialize/parallel_workload/executor.py b/misc/python/materialize/parallel_workload/executor.py index 052f9c0ab8dc9..11bbaf6c052e0 100644 --- a/misc/python/materialize/parallel_workload/executor.py +++ b/misc/python/materialize/parallel_workload/executor.py @@ -53,10 +53,16 @@ class Executor: reconnect_next: bool rollback_next: bool last_log: str + # "running" exactly while this session waits on the server. Every path that + # makes a round trip has to set it, the end-of-run wedge check reads it to + # tell a server-side hang from a worker stuck in the workload's own code. last_status: str action_run_since_last_commit_rollback: bool autocommit: bool user: str + # The session's transaction isolation, as last set through set_isolation. + # Actions that set an isolation transiently restore this one afterwards. + isolation: str def __init__( self, @@ -84,9 +90,17 @@ def __init__( self.use_ws = self.rng.choice([True, False]) if self.ws else False self.autocommit = cur.connection.autocommit self.mz_service = "materialized" + # Set while a non-default statement_timeout is configured on this + # session, statement timeouts are expected errors then. Cleared again + # on RESET and on reconnect, otherwise a hang in this worker stays + # invisible for the rest of the run. + self.statement_timeout_set = False + # Materialize's default, until the worker sets one on connect. + self.isolation = "STRICT SERIALIZABLE" def set_isolation(self, level: str) -> None: self.execute(f"SET TRANSACTION_ISOLATION TO '{level}'") + self.isolation = level def commit(self, http: Http = Http.RANDOM) -> None: self._end_transaction("commit", http) @@ -97,6 +111,7 @@ def rollback(self, http: Http = Http.RANDOM) -> None: def _end_transaction(self, command: str, http: Http) -> None: self.insert_table = None self.log(command) + self.last_status = "running" ws_error = None try: # When this executor uses the WS session, statements executed with @@ -118,6 +133,8 @@ def _end_transaction(self, command: str, http: Http) -> None: raise except Exception as e: raise QueryError(str(e), command) + finally: + self.last_status = "finished" if ws_error is not None: raise ws_error # TODO(def-): Enable when things are stable @@ -182,6 +199,7 @@ def copy( ) -> None: query += ";" self.log(f"{query} ({rows})") + self.last_status = "running" try: try: @@ -195,6 +213,22 @@ def copy( finally: self.last_status = "finished" + def copy_to_stdout(self, query: str) -> None: + query += ";" + self.log(query) + self.last_status = "running" + try: + try: + with self.cur.copy(query.encode()) as copy: + for _ in copy: + pass + except Exception as e: + raise QueryError(str(e), query) + + self.action_run_since_last_commit_rollback = True + finally: + self.last_status = "finished" + def execute( self, query: str, @@ -207,7 +241,37 @@ def execute( http == Http.RANDOM and self.rng.choice([True, False]) ) or http == Http.YES if explainable and self.rng.choice([True, False]): - query = f"EXPLAIN OPTIMIZED PLAN AS VERBOSE TEXT FOR {query}" + if self.rng.random() < 0.1: + as_json = " AS JSON" if self.rng.choice([True, False]) else "" + query = f"EXPLAIN TIMESTAMP{as_json} FOR {query}" + else: + stage = self.rng.choice( + [ + "RAW PLAN", + "DECORRELATED PLAN", + "LOCALLY OPTIMIZED PLAN", + "OPTIMIZED PLAN", + "OPTIMIZED PLAN", + "OPTIMIZED PLAN", + "PHYSICAL PLAN", + ] + ) + modifiers = "" + if stage == "OPTIMIZED PLAN" and self.rng.random() < 0.3: + mods = self.rng.sample( + [ + "arity", + "join implementations", + "keys", + "types", + "humanized expressions", + "redacted", + ], + self.rng.randint(1, 3), + ) + modifiers = f" WITH ({', '.join(mods)})" + format = self.rng.choice(["VERBOSE TEXT", "TEXT", "JSON"]) + query = f"EXPLAIN {stage}{modifiers} AS {format} FOR {query}" query += ";" extra_info_str = f" ({extra_info})" if extra_info else "" use_ws = self.use_ws and http != Http.NO diff --git a/misc/python/materialize/parallel_workload/expression.py b/misc/python/materialize/parallel_workload/expression.py index 72daf7e02d252..0d75239f02f16 100644 --- a/misc/python/materialize/parallel_workload/expression.py +++ b/misc/python/materialize/parallel_workload/expression.py @@ -45,6 +45,7 @@ from materialize.parallel_workload.column import ( Column, KafkaColumn, + LoadGeneratorColumn, MySqlColumn, PostgresColumn, SqlServerColumn, @@ -340,11 +341,33 @@ def __init__(self, text: str, params: list, unsupported: ExprKind = ExprKind.ALL # ] +# Edge-case literals injected at low probability as leaves of read/filter +# expressions (ExprKind.ALL only), to surface NaN/Infinity handling in +# arithmetic, comparisons, and aggregation. The generator's `Float` is float4, +# so these MUST be float4: a wider literal (float8/int8) injected where a float4 +# is expected breaks function overload resolution (e.g. round(numeric, bigint)), +# producing spurious failures rather than findings. +# +# Scope is deliberately narrow: NaN/Infinity are the genuinely-new coverage +# (random_value never produces them). Numeric/int boundary and overflow values +# are NOT injected here, because the existing LARGE record size already +# generates overflow-inducing magnitudes and mis-widthed literals only break +# overload resolution. Extreme-year date/timestamp literals are also omitted: +# a 6-digit year deterministically trips the known SS-345 date-parser bug in +# every context, spamming a known-unfixed issue rather than finding new ones. +EDGE_VALUES: dict[type, list[str]] = { + Float: ["'NaN'::float4", "'Infinity'::float4", "'-Infinity'::float4"], +} + + def expression( data_type: type[DataType], columns: list[Column] | ( list[MySqlColumn] - | (list[PostgresColumn] | (list[SqlServerColumn] | list[KafkaColumn])) + | ( + list[PostgresColumn] + | (list[SqlServerColumn] | (list[KafkaColumn] | list[LoadGeneratorColumn])) + ) ), rng: random.Random, kind: ExprKind = ExprKind.ALL, @@ -371,6 +394,12 @@ def expression( if col.data_type == data_type: return str(col) + # Only in read/filter contexts: overflow/eval errors are tolerated there, + # but not in write (INSERT value) or materialized-view-body contexts. + edges = EDGE_VALUES.get(data_type) + if edges and kind == ExprKind.ALL and rng.random() < 0.2: + return rng.choice(edges) + record_size = rng.choice( [RecordSize.TINY, RecordSize.SMALL, RecordSize.MEDIUM, RecordSize.LARGE] ) diff --git a/misc/python/materialize/parallel_workload/negative_accumulation_errors.py b/misc/python/materialize/parallel_workload/negative_accumulation_errors.py index 715ce5ba2704e..6f60be9596676 100644 --- a/misc/python/materialize/parallel_workload/negative_accumulation_errors.py +++ b/misc/python/materialize/parallel_workload/negative_accumulation_errors.py @@ -31,8 +31,18 @@ "Negative multiplicities in TopK", # Reduce "Net-zero records with non-zero accumulation in ReduceAccumulable", + # Client-facing variant of the above (reduce.rs:1514, EvalError), distinct + # from the internal ReduceAccumulable log text. Seen in repeat_row (#8106). + "with non-zero accumulation in accumulable aggregate", "Non-positive multiplicity in DistinctBy", + # Covers the `ReduceInaccumulable`, `ReduceInaccumulable DISTINCT` and + # `ReduceMinsMaxes` sites, which surface their internal log text verbatim. "Non-positive accumulation", + # The hierarchical min/max stage words its client-facing error differently + # from its "Non-positive accumulation in MinsMaxesHierarchical" log line + # (reduce.rs `build_bucketed_stage`), so the entry above does not cover it. + # Seen in repeat_row (build 17664). + "saw non-positive accumulation", "Invalid negative unsigned aggregation in ReduceAccumulable", "saw negative accumulation", # Peek handling @@ -43,6 +53,9 @@ "S3 oneshot sink encountered negative multiplicities", # Constant folding "Negative multiplicity in constant result", + # Constant folding a DISTINCT/INTERSECT/reduce over a repeat_row collection + # with negative diffs. Seen in repeat_row (builds 17205, 17214). + "constant folding encountered reduce on collection with non-positive multiplicities", # Scalar subquery guard "negative number of rows produced in subquery", ] diff --git a/misc/python/materialize/parallel_workload/parallel_workload.py b/misc/python/materialize/parallel_workload/parallel_workload.py index fb24101ca83af..4a2740fc70842 100644 --- a/misc/python/materialize/parallel_workload/parallel_workload.py +++ b/misc/python/materialize/parallel_workload/parallel_workload.py @@ -9,6 +9,7 @@ import argparse import datetime +import faulthandler import gc import os import random @@ -24,12 +25,17 @@ from materialize.parallel_workload.action import ( Action, ActionList, + ApplyReplacementMaterializedViewAction, BackupRestoreAction, CancelAction, + CreateReplacementMaterializedViewAction, DropClusterAction, DropDatabaseAction, + DropRoleAction, DropSchemaAction, + ExplainAnalyzeAction, KillAction, + SealedCollectionCheckAction, StatisticsAction, ZeroDowntimeDeployAction, action_lists, @@ -45,6 +51,7 @@ MAX_KAFKA_SINKS, MAX_KAFKA_SOURCES, MAX_MYSQL_SOURCES, + MAX_NETWORK_POLICIES, MAX_POSTGRES_SOURCES, MAX_ROLES, MAX_SCHEMAS, @@ -67,6 +74,23 @@ REPORT_TIME = 10 +def run_final_sealed_check( + rng: random.Random, database: Database, cur: psycopg.Cursor +) -> None: + """Guaranteed final oracle pass, before the objects are dropped. + + The finalize task seals a wrongly finalized shard only ~5s after the + triggering DROP, so damage from the run's last actions is not visible + immediately, and a run whose random scheduling never picked + SealedCollectionCheckAction would otherwise end without any pass. + """ + exe = Executor(rng, cur, None, database) + sealed_check = SealedCollectionCheckAction(rng, None) + if sealed_check.applicable(exe): + time.sleep(10) + sealed_check.run(exe) + + def run( host: str, ports: dict[str, int], @@ -123,6 +147,9 @@ def run( system_exe.execute( f"ALTER SYSTEM SET max_roles = {MAX_ROLES * 1000 + num_threads}" ) + system_exe.execute( + f"ALTER SYSTEM SET max_network_policies = {MAX_NETWORK_POLICIES + num_threads}" + ) system_exe.execute( f"ALTER SYSTEM SET max_clusters = {MAX_CLUSTERS * 40 + num_threads}" ) @@ -163,7 +190,7 @@ def run( system_exe.execute("DROP CLUSTER quickstart CASCADE") replica_names = [f"r{replica_id}" for replica_id in range(0, replicas)] replica_string = ",".join( - f"{replica_name} (SIZE 'scale=1,workers=4')" + f"{replica_name} (SIZE 'scale=1,workers=1')" for replica_name in replica_names ) system_exe.execute( @@ -228,10 +255,17 @@ def run( ) workers.append(worker) + # Daemon threads, so that a worker still stuck in a long-running query + # cannot block interpreter shutdown. The clean paths below join + # explicitly, and a WorkerFailedException propagates out of `run` past + # those joins. Without this a single wedged worker keeps the process + # alive until the CI step timeout kills it, which reports the timeout + # instead of the failure that actually happened. thread = threading.Thread( name=thread_name, target=worker.run, args=(host, ports["materialized"], ports["http"], "materialize", database), + daemon=True, ) thread.start() threads.append(thread) @@ -252,6 +286,7 @@ def run( name="cancel", target=worker.run, args=(host, ports["mz_system"], ports["http"], "mz_system", database), + daemon=True, ) thread.start() threads.append(thread) @@ -272,6 +307,7 @@ def run( name="kill", target=worker.run, args=(host, ports["materialized"], ports["http"], "materialize", database), + daemon=True, ) thread.start() threads.append(thread) @@ -299,6 +335,7 @@ def run( name="zero-downtime-deploy", target=worker.run, args=(host, ports["materialized"], ports["http"], "materialize", database), + daemon=True, ) thread.start() threads.append(thread) @@ -319,6 +356,7 @@ def run( name="kill", target=worker.run, args=(host, ports["materialized"], ports["http"], "materialize", database), + daemon=True, ) thread.start() threads.append(thread) @@ -343,6 +381,7 @@ def run( name="statistics", target=worker.run, args=(host, ports["mz_system"], ports["http"], "mz_system", database), + daemon=True, ) thread.start() threads.append(thread) @@ -379,13 +418,79 @@ def run( if all([not thread.is_alive() for thread in threads]): break else: - for worker, thread in zip(workers, threads): - if thread.is_alive(): - print( - f"{thread.name} still running ({worker.exe.mz_service}): {worker.exe.last_log} ({worker.exe.last_status})" - ) + alive = [(w, t) for w, t in zip(workers, threads) if t.is_alive()] + for worker, thread in alive: + if worker.exe is None: + # Still in the initial connect retry loop. + print(f"{thread.name} still connecting") + continue + print( + f"{thread.name} still running ({worker.exe.mz_service}): {worker.exe.last_log} ({worker.exe.last_status})" + ) + + # Workers are daemon threads, so a wedged one cannot keep the process + # alive and has to be caught here or it silently costs the run its + # remaining workload. A worker waiting on the server is a server-side + # hang (DB-118) and is tolerated, as is one still in its connect retry + # loop (exe is None), which can only be hanging inside a connect call + # to the server. One that never waits on the server is stuck in the + # workload's own code, e.g. deadlocked on two object locks taken in + # opposite orders, which no timeout ever clears. Nothing legitimate + # pauses without a round trip for the 300s above (the longest such + # pause is BackupRestoreAction's 240s sleep), but a single sample can + # catch a worker between round trips, so only call a worker wedged if + # it stays off the server for a whole confirmation window. + # + # A tolerated server-side hang also explains every other worker: the + # hung statement holds its actions' object locks (an `ALTER SINK ... + # SET FROM` waits unboundedly for the sink's frontier to catch up, + # database-issues#9820, while holding both the sink's and the new base + # object's lock), and a worker queued on such a lock can no more reach + # the server than a deadlocked one can. Telling the two apart from + # outside is not possible, so a workload-side deadlock is only + # diagnosed when no worker is on the server at all. Waiters and + # deadlocks both resolve when the statement does, and calling a waiter + # deadlocked costs the run its final checks. + def not_on_server() -> set[str]: + return { + t.name + for w, t in alive + if t.is_alive() and w.exe is not None and w.exe.last_status != "running" + } + + def on_server() -> set[str]: + return { + t.name + for w, t in alive + if t.is_alive() and w.exe is not None and w.exe.last_status == "running" + } + + wedged = not_on_server() + confirm_until = time.time() + 60 + while wedged and time.time() < confirm_until: + time.sleep(1) + wedged &= not_on_server() + if on_server(): + wedged = set() + break merge_num_queries(num_queries, workers) - print_stats(num_queries, workers, num_threads, scenario) + print_stats(num_queries, workers, num_threads, complexity, scenario) + + # A wedged worker must not mask damage, so run the final oracle pass + # on a fresh connection before exiting. Skipped for 0dt deploys, + # where connecting to the fenced-out environmentd can hang forever. + if scenario != Scenario.ZeroDowntimeDeploy: + try: + check_conn = psycopg.connect( + host=host, port=ports["materialized"], user="materialize" + ) + except Exception as e: + print(f"Skipping final sealed-collection check: {e}") + else: + check_conn.autocommit = True + with check_conn.cursor() as cur: + run_final_sealed_check(rng, database, cur) + check_conn.close() if num_threads >= 50: # Under high load some queries can't finish quickly, especially UPDATE/DELETE @@ -395,9 +500,13 @@ def run( # environmentd will be stuck forever, the promoted environmentd can # take > 10 minutes to become responsive as well os._exit(0) - # TODO: Reenable when https://linear.app/materializeinc/issue/DB-118 is fixed - # print("Threads have not stopped within 5 minutes, exiting hard") - # os._exit(1) + if wedged: + faulthandler.dump_traceback() + print( + "^^^ +++ Threads have not stopped within 5 minutes and are not" + f" waiting on the server, exiting hard: {', '.join(sorted(wedged))}" + ) + os._exit(1) os._exit(0) try: @@ -422,6 +531,8 @@ def run( # increment can still be in flight. database.read_then_write_counter.validate(exe) + run_final_sealed_check(rng, database, cur) + # Dropping the database also releases the long running connections # used by database objects. database.drop(exe) @@ -446,7 +557,7 @@ def run( conn.close() merge_num_queries(num_queries, workers) - print_stats(num_queries, workers, num_threads, scenario) + print_stats(num_queries, workers, num_threads, complexity, scenario) def merge_num_queries( @@ -474,6 +585,7 @@ def print_stats( num_queries: defaultdict[ActionList, Counter[type[Action]]], workers: list[Worker], num_threads: int, + complexity: Complexity, scenario: Scenario, ) -> None: ignored_errors: defaultdict[str, Counter[type[Action]]] = defaultdict(Counter) @@ -527,7 +639,11 @@ def print_stats( # (ReplaceMaterializedView, whose replacements pile up unfinalized in the # workload). "unknown cluster 'dont_exist'" comes from FlipFlags setting the # default cluster to a nonexistent one to hunt for panics, so any action - # needing a cluster fails with it while that flag is live. Tracked + # needing a cluster fails with it while that flag is live. "cannot be + # dropped because some objects depend on it" is the same rejection for + # DROP ROLE: AlterOwnerAction reassigns object ownership to random roles, + # so a role usually owns something and a short run can see DropRoleAction + # never land a dependency-free role. Tracked # separately so such actions don't trip the broken-action assertion below. noise = { "must be owner of", @@ -537,14 +653,26 @@ def print_stats( "because it already has a replacement", "is sealed and thus cannot be replaced", "unknown cluster 'dont_exist'", + "cannot be dropped because some objects depend on it", } + if complexity in (Complexity.DDL, Complexity.DDLOnly): + # CreateTableAction and CreateViewAction make half their objects TEMP, + # and those land in the shared table and view lists every worker samples + # from. A temporary schema belongs to the session that created it, so + # any other worker resolving "db"."mz_temp"."name" fails with "unknown + # schema 'mz_temp'". That is roughly half the attempts of an action + # picking a random table or view, which an action as low-weight as + # RevokePrivilegesAction can spend a whole run on. Concurrent schema + # drops and renames land here too. Only DDL complexity creates temp + # objects or touches schemas, so elsewhere this stays a genuine signal. + noise.add("unknown schema") if scenario == Scenario.Rename: # Concurrent renames invalidate the qualified names an action captured # earlier (a stored SELECT it re-renders, a target it resolved before # the rename), so a rare action can fail on every attempt with a # name-resolution race. Expected only in this scenario. In Regression, # where nothing renames, these would be a genuine signal. - noise |= {"does not exist", "unknown schema", "unknown catalog item"} + noise |= {"does not exist", "unknown catalog item"} num_errored_real: Counter[type[Action]] = Counter() for worker in workers: num_successes.update(worker.num_successes) @@ -558,11 +686,42 @@ def print_stats( for action_list in action_lists for action_class in action_list.action_classes } - # These use RESTRICT and their targets practically always contain - # objects (schemas and databases their items, clusters their sources, - # sinks, and indexes), so they exercise the rejection path and are not - # expected to ever succeed. - action_classes -= {DropClusterAction, DropDatabaseAction, DropSchemaAction} + # A given churny seed can legitimately see zero successes for these, so the + # broken-action assertion must not fire on them: + # * DropCluster/DropDatabase/DropSchema: RESTRICT rejections. Their + # targets practically always contain objects, and the rejection message + # is class-specific, so `noise` above cannot cover it without excusing + # it for every other action too. + # * DropRoleAction: needs a dependency-free role, but AlterOwnerAction + # keeps reassigning object ownership to random roles. + # * ExplainAnalyzeAction: needs a hydrated MV/index on the active cluster, + # which renames/drops keep retiring. + # These succeed in normal runs, so they aren't broken. + # + # NOTE: listed by name rather than matched on a Drop* name prefix. The other + # Drop* actions pick an existing object and are expected to succeed + # constantly, and a broken one (wrong quoting or qualification) fails with + # `unknown catalog item`, which the base errors_to_ignore tolerates and + # `noise` does not list. That is exactly what the assertion is meant to + # catch, so a prefix would silently exempt the failure mode it exists for. + excluded: set[type[Action]] = { + DropClusterAction, + DropDatabaseAction, + DropRoleAction, + DropSchemaAction, + ExplainAnalyzeAction, + } + if scenario == Scenario.Rename: + # CreateReplacementMaterializedViewAction re-renders the view's SELECT + # with the object names captured at creation. Renames invalidate them, + # so CREATE REPLACEMENT fails with a tolerated "does not exist"/"unknown + # schema", and a churny rename seed can legitimately never land a clean + # attempt. With no replacement ever created, + # ApplyReplacementMaterializedViewAction then has nothing to apply + # either. Both succeed in the other scenarios, so they stay checked + # there. + excluded.add(CreateReplacementMaterializedViewAction) + excluded.add(ApplyReplacementMaterializedViewAction) never_succeeded = [] for action_class in sorted(action_classes, key=lambda cls: cls.__name__): successes = num_successes[action_class] @@ -596,7 +755,7 @@ def print_stats( always_erroring = [ action_class.__name__ for action_class, skips, errored in never_succeeded - if num_errored_real[action_class] > 0 + if num_errored_real[action_class] > 0 and action_class not in excluded ] assert ( not always_erroring @@ -636,7 +795,12 @@ def parse_common_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--azurite", action="store_true", help="Use Azurite as blob store instead of S3" ) - parser.add_argument("--replicas", type=int, default=2, help="use multiple replicas") + parser.add_argument( + "--replicas", + type=int, + default=1, + help="default replica number for quickstart cluster", + ) def main() -> int: diff --git a/misc/python/materialize/parallel_workload/settings.py b/misc/python/materialize/parallel_workload/settings.py index 79196897d521a..4efdd32a3876e 100644 --- a/misc/python/materialize/parallel_workload/settings.py +++ b/misc/python/materialize/parallel_workload/settings.py @@ -58,4 +58,28 @@ def _missing_(cls, value): # it on outside that scenario is harmless: no Parallel Workload codegen # emits `repeat_row` unless the scenario is active. "enable_repeat_row": "true", + # TODO: Reenable once the frontend-peek path stops acquiring a read hold + # whose `since` has compacted past the chosen `as_of`. The soft-assert + # assert_read_holds_correct (frontend_peek.rs:1759) fires for peeks in + # multi-statement transactions ("... read hold at .. is not enough for + # as_of .."), panicking the coordinator. Peeks still work via the classic + # coordinator path with this off. + # See https://linear.app/materializeinc/issue/SQL-520 + "enable_frontend_peek_sequencing": "false", + # 64 MiB, down from the 1 GiB default. A peek's result is materialized in + # memory on the replica serving it before this bound errors it + # (`to_error_if_exceeds`, compute_state.rs), and measured on a workload-like + # relation a 300 MiB result costs ~1 GiB of replica RSS plus ~0.5 GiB in + # environmentd. The ceiling is per in-flight statement and every worker + # holds a pg and a WebSocket session, so at the default a handful of + # concurrent wide reads can claim tens of GiB on the default cluster, where + # every peek lands. All containers of a run share one cgroup budget (24 GiB + # on the CI agent, environmentd plus every replica process plus Kafka, + # Postgres, MySQL, SQL Server), so that ends in the kernel OOM-killer + # picking a victim and taking unrelated processes down with it. Runs do + # reach the cap ("result exceeds max size of" shows up in the ignored-error + # statistics), which is what makes the tail dangerous. Query shapes are + # unaffected: an oversized result errors instead of being buffered, and that + # error is already tolerated for every action. + "max_result_size": "67108864", }