Skip to content

Commit

Permalink
allow type casting in static schema (parseablehq#729)
Browse files Browse the repository at this point in the history
if field is defined as float,
below values (examples) can be accepted -
100,-100.45, 100.23, "200.45"
  • Loading branch information
nikhilsinhaparseable committed Apr 19, 2024
1 parent df1ef5e commit b196039
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 5 deletions.
8 changes: 6 additions & 2 deletions server/src/event/format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ pub trait EventFormat: Sized {
self,
schema: HashMap<String, Arc<Field>>,
time_partition: Option<String>,
static_schema_flag: Option<String>,
) -> Result<(Self::Data, EventSchema, bool, Tags, Metadata), AnyError>;
fn decode(data: Self::Data, schema: Arc<Schema>) -> Result<RecordBatch, AnyError>;
fn into_recordbatch(
Expand All @@ -50,8 +51,11 @@ pub trait EventFormat: Sized {
time_partition: Option<String>,
static_schema_flag: Option<String>,
) -> Result<(RecordBatch, bool), AnyError> {
let (data, mut schema, is_first, tags, metadata) =
self.to_data(storage_schema.clone(), time_partition)?;
let (data, mut schema, is_first, tags, metadata) = self.to_data(
storage_schema.clone(),
time_partition,
static_schema_flag.clone(),
)?;

if get_field(&schema, DEFAULT_TAGS_KEY).is_some() {
return Err(anyhow!("field {} is a reserved field", DEFAULT_TAGS_KEY));
Expand Down
8 changes: 5 additions & 3 deletions server/src/event/format/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ impl EventFormat for Event {
self,
schema: HashMap<String, Arc<Field>>,
time_partition: Option<String>,
static_schema_flag: Option<String>,
) -> Result<(Self::Data, Vec<Arc<Field>>, bool, Tags, Metadata), anyhow::Error> {
let data = flatten_json_body(self.data, time_partition)?;
let stream_schema = schema;
Expand Down Expand Up @@ -91,9 +92,10 @@ impl EventFormat for Event {
},
};

if value_arr
.iter()
.any(|value| fields_mismatch(&schema, value))
if static_schema_flag.is_none()
&& value_arr
.iter()
.any(|value| fields_mismatch(&schema, value))
{
return Err(anyhow!(
"Could not process this event due to mismatch in datatype"
Expand Down

0 comments on commit b196039

Please sign in to comment.