Skip to content

Commit fa8ae9d

Browse files
dnagodadnagoda
committed
Build the readable input and output text only when it is shown
`BytesContainer::new` pretty-printed every input and output into a `humanized` string, and `run` deep-copied the input container for the I/O handler. Batch mode never shows that text, so for large inputs most of the time per input went to building and copying data that was then dropped. - Remove the `humanized` field and implement `Display` for `BytesContainer`. The text is built only when a result is displayed. - Move parsed JSON values into the container instead of cloning them. - `IOHandler` borrows the input instead of taking a deep copy. - Pass the Function output bytes to the container without a copy. An output that is not valid JSON or MessagePack now keeps its raw bytes, so "Output Size" shows the real size instead of 0B.
1 parent 056ca6a commit fa8ae9d

5 files changed

Lines changed: 77 additions & 71 deletions

File tree

‎README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,17 @@ if its `engine` differs from the module's engine. The
111111
`prepared_runs_reuse_one_validated_module` test in `tests/library_api.rs`
112112
runs this example against a fixture Function.
113113

114+
`BytesContainer` implements `Display`. Use `container.to_string()` or
115+
`format!("{container}")` to get readable text for an input or an output:
116+
117+
- Pretty-printed JSON when the bytes decode to JSON or MessagePack.
118+
- A hex dump for the `Raw` codec.
119+
- The bytes as lossy UTF-8 when an output does not decode. `encoding_error`
120+
gives the reason, and `raw` keeps the bytes.
121+
122+
The text is built only when you display the container, so creating a
123+
container does not pay for it. This replaces the `humanized` field.
124+
114125
To compute scale factors for many inputs, use
115126
`bluejay_schema_analyzer::BluejaySchemaAnalyzer::with_analyzer`. It parses and
116127
validates the schema and query once, then calls your closure with an `analyze`

‎src/container.rs‎

Lines changed: 56 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use std::io;
1+
use std::{fmt, io};
22

33
use crate::Codec;
44
use anyhow::{anyhow, Result};
@@ -109,9 +109,6 @@ pub struct BytesContainer {
109109
/// The JSON represantation of the bytes.
110110
#[serde(flatten)]
111111
pub json_value: Option<serde_json::Value>,
112-
/// The human readable representation of the bytes.
113-
#[serde(skip)]
114-
pub humanized: String,
115112
/// Context for encoding errors.
116113
#[serde(skip)]
117114
pub encoding_error: Option<String>,
@@ -121,7 +118,6 @@ impl Default for BytesContainer {
121118
fn default() -> Self {
122119
Self {
123120
codec: Codec::Raw,
124-
humanized: "<raw codec>".into(),
125121
json_value: None,
126122
raw: Default::default(),
127123
encoding_error: None,
@@ -132,20 +128,11 @@ impl Default for BytesContainer {
132128
impl BytesContainer {
133129
pub fn new(ty: BytesContainerType, codec: Codec, raw: Vec<u8>) -> Result<Self> {
134130
match codec {
135-
Codec::Raw => {
136-
let humanized = raw
137-
.iter()
138-
.map(|b| format!("{:02x}", b))
139-
.collect::<Vec<String>>()
140-
.join(" ");
141-
142-
Ok(Self {
143-
raw,
144-
codec,
145-
humanized,
146-
..Default::default()
147-
})
148-
}
131+
Codec::Raw => Ok(Self {
132+
raw,
133+
codec,
134+
..Default::default()
135+
}),
149136
Codec::Json => match ty {
150137
BytesContainerType::Input => {
151138
let json = serde_json::from_slice::<serde_json::Value>(&raw)
@@ -155,30 +142,25 @@ impl BytesContainer {
155142
Ok(Self {
156143
codec,
157144
raw: minified_buffer,
158-
json_value: Some(json.clone()),
159-
humanized: serde_json::to_string_pretty(&json)?,
145+
json_value: Some(json),
160146
encoding_error: None,
161147
})
162148
}
163149
BytesContainerType::Output => {
164-
let mut this = Self {
165-
codec,
166-
..Default::default()
167-
};
168-
169150
match serde_json::from_slice::<serde_json::Value>(&raw) {
170-
Ok(json) => {
171-
this.json_value = Some(json.clone());
172-
this.humanized = serde_json::to_string_pretty(&json)?;
173-
this.raw = serde_json::to_vec(&json)?;
174-
}
175-
Err(e) => {
176-
this.humanized = String::from_utf8_lossy(&raw).into();
177-
this.encoding_error = Some(e.to_string());
178-
}
179-
};
180-
181-
Ok(this)
151+
Ok(json) => Ok(Self {
152+
codec,
153+
raw: serde_json::to_vec(&json)?,
154+
json_value: Some(json),
155+
encoding_error: None,
156+
}),
157+
Err(e) => Ok(Self {
158+
codec,
159+
raw,
160+
json_value: None,
161+
encoding_error: Some(e.to_string()),
162+
}),
163+
}
182164
}
183165
},
184166
Codec::Messagepack => match ty {
@@ -191,37 +173,54 @@ impl BytesContainer {
191173
Ok(Self {
192174
raw: bytes,
193175
codec,
194-
json_value: Some(json.clone()),
195-
humanized: serde_json::to_string_pretty(&json)?,
176+
json_value: Some(json),
196177
encoding_error: None,
197178
})
198179
}
199180
BytesContainerType::Output => {
200-
let mut this = Self {
201-
codec,
202-
..Default::default()
203-
};
204-
205181
let value: Result<serde_json::Value, _> = rmp_serde::decode::from_slice(&raw);
206182
match value {
207-
Ok(json) => {
208-
this.json_value = Some(json.clone());
209-
this.humanized = serde_json::to_string_pretty(&json)?;
210-
this.raw = raw;
211-
}
212-
Err(e) => {
213-
this.humanized = String::from_utf8_lossy(&raw).into();
214-
this.encoding_error = Some(e.to_string());
215-
}
216-
};
217-
218-
Ok(this)
183+
Ok(json) => Ok(Self {
184+
codec,
185+
raw,
186+
json_value: Some(json),
187+
encoding_error: None,
188+
}),
189+
Err(e) => Ok(Self {
190+
codec,
191+
raw,
192+
json_value: None,
193+
encoding_error: Some(e.to_string()),
194+
}),
195+
}
219196
}
220197
},
221198
}
222199
}
223200
}
224201

202+
impl fmt::Display for BytesContainer {
203+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
204+
if let Codec::Raw = self.codec {
205+
for (index, byte) in self.raw.iter().enumerate() {
206+
if index > 0 {
207+
f.write_str(" ")?;
208+
}
209+
write!(f, "{byte:02x}")?;
210+
}
211+
return Ok(());
212+
}
213+
214+
match &self.json_value {
215+
Some(json) => {
216+
let pretty = serde_json::to_string_pretty(json).map_err(|_| fmt::Error)?;
217+
f.write_str(&pretty)
218+
}
219+
None => f.write_str(&String::from_utf8_lossy(&self.raw)),
220+
}
221+
}
222+
}
223+
225224
#[cfg(test)]
226225
mod tests {
227226
use super::*;

‎src/engine.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ pub fn run_prepared(params: PreparedFunctionRunParams) -> Result<FunctionRunResu
134134
} = params;
135135

136136
let engine = module.inner().engine();
137-
let mut io_handler = IOHandler::new(module, input.clone());
137+
let mut io_handler = IOHandler::new(module, &input);
138138

139139
let mut error_logs: String = String::new();
140140

@@ -200,11 +200,7 @@ pub fn run_prepared(params: PreparedFunctionRunParams) -> Result<FunctionRunResu
200200
logs.extend_from_slice(error_logs.as_bytes());
201201

202202
let output_codec = input.codec;
203-
let output = BytesContainer::new(
204-
BytesContainerType::Output,
205-
output_codec,
206-
raw_output.to_vec(),
207-
)?;
203+
let output = BytesContainer::new(BytesContainerType::Output, output_codec, raw_output)?;
208204

209205
let name = function_path.file_name().unwrap().to_str().unwrap();
210206
let size = function_path.metadata()?.len() / 1024;

‎src/function_run_result.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ impl fmt::Display for FunctionRunResult {
8585
formatter,
8686
"{}\n\n{}",
8787
" Input ".black().on_bright_yellow(),
88-
self.input.humanized,
88+
self.input,
8989
)?;
9090

9191
writeln!(
@@ -111,7 +111,7 @@ impl fmt::Display for FunctionRunResult {
111111
formatter,
112112
"{}\n\n{}",
113113
" Invalid Output ".black().on_bright_red(),
114-
self.output.humanized,
114+
self.output,
115115
)?;
116116

117117
writeln!(
@@ -125,7 +125,7 @@ impl fmt::Display for FunctionRunResult {
125125
formatter,
126126
"{}\n\n{}",
127127
" Output ".black().on_bright_green(),
128-
self.output.humanized,
128+
self.output,
129129
)?;
130130
}
131131

@@ -250,7 +250,7 @@ mod tests {
250250

251251
let predicate = predicates::str::contains("Instructions: 1.001K")
252252
.and(predicates::str::contains("Linear Memory Usage: 1000KB"))
253-
.and(predicates::str::contains(input.humanized))
253+
.and(predicates::str::contains(input.to_string()))
254254
.and(predicates::str::contains("Input Size: 28B"))
255255
.and(predicates::str::contains("Output Size: 15B"));
256256
assert!(predicate.eval(&function_run_result.to_string()));
@@ -280,7 +280,7 @@ mod tests {
280280

281281
let predicate = predicates::str::contains("Instructions: 1")
282282
.and(predicates::str::contains("Linear Memory Usage: 1000KB"))
283-
.and(predicates::str::contains(input.humanized));
283+
.and(predicates::str::contains(input.to_string()));
284284
assert!(predicate.eval(&function_run_result.to_string()));
285285
Ok(())
286286
}
@@ -306,7 +306,7 @@ mod tests {
306306

307307
let predicate = predicates::str::contains("Instructions: 999")
308308
.and(predicates::str::contains("Linear Memory Usage: 1000KB"))
309-
.and(predicates::str::contains(input.humanized));
309+
.and(predicates::str::contains(input.to_string()));
310310
assert!(predicate.eval(&function_run_result.to_string()));
311311
Ok(())
312312
}

‎src/io.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ enum IOStrategy {
2828
pub(crate) struct IOHandler<'a> {
2929
strategy: IOStrategy,
3030
module: &'a ValidatedModule,
31-
input: BytesContainer,
31+
input: &'a BytesContainer,
3232
}
3333

3434
impl<'a> IOHandler<'a> {
35-
pub(crate) fn new(module: &'a ValidatedModule, input: BytesContainer) -> Self {
35+
pub(crate) fn new(module: &'a ValidatedModule, input: &'a BytesContainer) -> Self {
3636
Self {
3737
strategy: if module.uses_mem_io() {
3838
IOStrategy::Memory(None)

0 commit comments

Comments
 (0)