Skip to content

Update build process to add "null" as an allowed type for non-required properties #811

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Update build process to add "null" as an allowed type for non-required properties #811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a9420aa
Update build process to add "null" as an allowed type for non-require…
sean-rose May 2, 2024
6becf29
Rebuild schemas to add "null" as an allowed type for non-required pro…
sean-rose May 2, 2024
010dd27
Adjust tests.
sean-rose May 2, 2024
e54bdd9
Update `is_valid_telemetry_version()` to avoid erroring on nullable `…
sean-rose May 3, 2024
4fdc98a
Refactor `allow_null_nonrequired_properties()`.
sean-rose May 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0")

find_program(PYTHON_EXE python3)
if(NOT PYTHON_EXE)
message(FATAL_ERROR "python3 is required for output formatting and metadata injection")
message(FATAL_ERROR "python3 is required for output formatting and schema transformation")
endif()

find_program(PARQUETFMT_EXE parquetfmt)
Expand Down Expand Up @@ -67,12 +67,12 @@ foreach(ITEM IN ITEMS ${SCHEMAS})
# NOTE the artifacts are written back into the source repository (see the README)
set(IN_FILE "${CMAKE_SOURCE_DIR}/templates/${ITEM}")
set(OUT_FILE "${CMAKE_SOURCE_DIR}/schemas/${ITEM}")
if(ITEM MATCHES "\\.json$") # inject metadata and pretty print
if(ITEM MATCHES "\\.json$") # transform schema and pretty print
set(TMP_FILE "${OUT_FILE}.tmp")
configure_file(${IN_FILE} ${TMP_FILE} @ONLY)
execute_process(COMMAND ${PYTHON_EXE} ../scripts/inject-metadata ${ITEM} RESULT_VARIABLE INJECT_RV INPUT_FILE ${TMP_FILE} OUTPUT_FILE ${OUT_FILE})
if(NOT INJECT_RV EQUAL 0)
message(FATAL_ERROR "python output error ${INJECT_RV} ${ITEM}")
execute_process(COMMAND ${PYTHON_EXE} ../scripts/transform-schema ${ITEM} RESULT_VARIABLE TRANSFORM_RV INPUT_FILE ${TMP_FILE} OUTPUT_FILE ${OUT_FILE})
if(NOT TRANSFORM_RV EQUAL 0)
message(FATAL_ERROR "python output error ${TRANSFORM_RV} ${ITEM}")
endif()
file(REMOVE ${TMP_FILE}) # the tmp file is intentionally not removed on failure so it can be examined
elseif(ITEM MATCHES "\\.parquetmr\\.txt$")
Expand Down
63 changes: 50 additions & 13 deletions schemas/activity-stream/events/events.1.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,20 @@
"properties": {
"action_position": {
"description": "A zero based integer indicating the position of this event",
"type": "integer"
"type": [
"integer",
"null"
]
},
"addon_version": {
"type": "string"
},
"browser_session_id": {
"description": "A mirror of the browser sessionId, as defined in https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/main/schemas/telemetry/main/main.4.schema.json",
"type": "string"
"type": [
"string",
"null"
]
},
"client_id": {
"pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
Expand All @@ -42,7 +48,10 @@
"type": "object"
},
"description": "An object to record all active experiments, experiments IDs are stored as keys, and the value object stores the branch information. Example: {\"experiment_1\": {\"branch\": \"control\"}, \"experiment_2\": {\"branch\": \"treatment\"}}. This deprecates the \"shield_id\" used in activity-stream and messaging-system.",
"type": "object"
"type": [
"object",
"null"
]
},
"locale": {
"type": "string"
Expand All @@ -55,38 +64,66 @@
"unknown",
"n/a",
"both",
"https://newtab.firefoxchina.cn/newtab/as/activity-stream.html"
"https://newtab.firefoxchina.cn/newtab/as/activity-stream.html",
null
],
"type": "string"
"type": [
"string",
"null"
]
},
"profile_creation_date": {
"type": "integer"
"type": [
"integer",
"null"
]
},
"region": {
"type": "string"
"type": [
"string",
"null"
]
},
"release_channel": {
"type": "string"
"type": [
"string",
"null"
]
},
"session_id": {
"description": "A UUID representing an Activity Stream session. This can be used to do table joins between `sessions` and `events` in Activity Stream. Note that `n/a` denotes that the session is not applicable in the context.",
"pattern": "^\\{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\}$|^n/a$",
"type": "string"
"type": [
"string",
"null"
]
},
"shield_id": {
"description": "[DEPRECATED]: use `experiments` instead. A semicolon separated string to store a list of Shield study IDs",
"type": "string"
"type": [
"string",
"null"
]
},
"source": {
"type": "string"
"type": [
"string",
"null"
]
},
"user_prefs": {
"description": "An encoded integer representing user's preferences of Activity Stream",
"type": "integer"
"type": [
"integer",
"null"
]
},
"value": {
"description": "A string that describes the context about this event",
"type": "string"
"type": [
"string",
"null"
]
},
"version": {
"type": "string"
Expand Down
70 changes: 56 additions & 14 deletions schemas/activity-stream/impression-stats/impression-stats.1.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,17 @@
},
"block": {
"description": "A 0-based index to record which tile in the 'tiles' list the user just interacted with",
"type": "integer"
"type": [
"integer",
"null"
]
},
"click": {
"description": "A 0-based index to record which tile in the 'tiles' list the user just interacted with",
"type": "integer"
"type": [
"integer",
"null"
]
},
"experiments": {
"additionalProperties": {
Expand All @@ -34,7 +40,10 @@
"type": "object"
},
"description": "An object to record all active experiments, experiments IDs are stored as keys, and the value object stores the branch information. Example: {\"experiment_1\": {\"branch\": \"control\"}, \"experiment_2\": {\"branch\": \"treatment\"}}. This deprecates the \"shield_id\" used in activity-stream and messaging-system.",
"type": "object"
"type": [
"object",
"null"
]
},
"impression_id": {
"description": "A UUID representing this user. Note that it's not client_id, nor can it be used to link to a client_id",
Expand All @@ -43,7 +52,10 @@
},
"loaded": {
"description": "An integer to record the size of the loaded 'tiles' list",
"type": "integer"
"type": [
"integer",
"null"
]
},
"locale": {
"type": "string"
Expand All @@ -62,23 +74,41 @@
},
"pocket": {
"description": "A 0-based index to record which tile in the 'tiles' list the user just interacted with",
"type": "integer"
"type": [
"integer",
"null"
]
},
"profile_creation_date": {
"type": "integer"
"type": [
"integer",
"null"
]
},
"region": {
"type": "string"
"type": [
"string",
"null"
]
},
"release_channel": {
"type": "string"
"type": [
"string",
"null"
]
},
"shield_id": {
"description": "[DEPRECATED]: use `experiments` instead. A semicolon separated string to store a list of Shield study IDs",
"type": "string"
"type": [
"string",
"null"
]
},
"source": {
"type": "string"
"type": [
"string",
"null"
]
},
"tiles": {
"items": {
Expand All @@ -87,7 +117,10 @@
"type": "integer"
},
"pos": {
"type": "integer"
"type": [
"integer",
"null"
]
}
},
"required": [
Expand All @@ -101,15 +134,24 @@
"type": "integer"
},
"version": {
"type": "string"
"type": [
"string",
"null"
]
},
"window_inner_height": {
"description": "The interior height of the window in pixels, including the height of the horizontal scroll bar, if present",
"type": "integer"
"type": [
"integer",
"null"
]
},
"window_inner_width": {
"description": "The interior width of the window in pixels. This includes the width of the vertical scroll bar, if one is present.",
"type": "integer"
"type": [
"integer",
"null"
]
}
},
"required": [
Expand Down
65 changes: 52 additions & 13 deletions schemas/activity-stream/impression-stats/impression-stats.2.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,17 @@
},
"block": {
"description": "A 0-based index to record which tile in the 'tiles' list the user just interacted with",
"type": "integer"
"type": [
"integer",
"null"
]
},
"click": {
"description": "A 0-based index to record which tile in the 'tiles' list the user just interacted with",
"type": "integer"
"type": [
"integer",
"null"
]
},
"experiments": {
"additionalProperties": {
Expand All @@ -34,7 +40,10 @@
"type": "object"
},
"description": "An object to record all active experiments, experiments IDs are stored as keys, and the value object stores the branch information. Example: {\"experiment_1\": {\"branch\": \"control\"}, \"experiment_2\": {\"branch\": \"treatment\"}}. This deprecates the \"shield_id\" used in activity-stream and messaging-system.",
"type": "object"
"type": [
"object",
"null"
]
},
"impression_id": {
"description": "A UUID representing this user. Note that it's not client_id, nor can it be used to link to a client_id",
Expand All @@ -43,7 +52,10 @@
},
"loaded": {
"description": "An integer to record the size of the loaded 'tiles' list",
"type": "integer"
"type": [
"integer",
"null"
]
},
"locale": {
"type": "string"
Expand All @@ -62,19 +74,34 @@
},
"pocket": {
"description": "A 0-based index to record which tile in the 'tiles' list the user just interacted with",
"type": "integer"
"type": [
"integer",
"null"
]
},
"profile_creation_date": {
"type": "integer"
"type": [
"integer",
"null"
]
},
"region": {
"type": "string"
"type": [
"string",
"null"
]
},
"release_channel": {
"type": "string"
"type": [
"string",
"null"
]
},
"source": {
"type": "string"
"type": [
"string",
"null"
]
},
"tiles": {
"items": {
Expand All @@ -84,7 +111,10 @@
"type": "string"
},
"pos": {
"type": "integer"
"type": [
"integer",
"null"
]
}
},
"required": [
Expand All @@ -98,15 +128,24 @@
"type": "integer"
},
"version": {
"type": "string"
"type": [
"string",
"null"
]
},
"window_inner_height": {
"description": "The interior height of the window in pixels, including the height of the horizontal scroll bar, if present",
"type": "integer"
"type": [
"integer",
"null"
]
},
"window_inner_width": {
"description": "The interior width of the window in pixels. This includes the width of the vertical scroll bar, if one is present.",
"type": "integer"
"type": [
"integer",
"null"
]
}
},
"required": [
Expand Down
Loading