Add inscription compression (ordinals#1713)

Cargo.toml

@@ -23,6 +23,7 @@ bech32 = "0.9.1"
 bip39 = "2.0.0"
 bitcoin = { version = "0.30.1", features = ["rand"] }
 boilerplate = { version = "1.0.0", features = ["axum"] }
+brotli = "3.4.0"
 chrono = "0.4.19"
 ciborium = "0.2.1"
 clap = { version = "4.4.2", features = ["derive"] }
@@ -63,7 +64,7 @@ tower-http = { version = "0.4.0", features = ["compression-br", "compression-gzi
 [dev-dependencies]
 executable-path = "1.0.0"
 pretty_assertions = "1.2.1"
-reqwest = { version = "0.11.10", features = ["blocking", "json"] }
+reqwest = { version = "0.11.10", features = ["blocking", "brotli", "json"] }
 test-bitcoincore-rpc = { path = "test-bitcoincore-rpc" }
 unindent = "0.2.1"
 

src/envelope.rs

@@ -14,6 +14,7 @@ pub(crate) const POINTER_TAG: [u8; 1] = [2];
 pub(crate) const PARENT_TAG: [u8; 1] = [3];
 pub(crate) const METADATA_TAG: [u8; 1] = [5];
 pub(crate) const METAPROTOCOL_TAG: [u8; 1] = [7];
+pub(crate) const CONTENT_ENCODING_TAG: [u8; 1] = [9];
 
 type Result<T> = std::result::Result<T, script::Error>;
 type RawEnvelope = Envelope<Vec<Vec<u8>>>;
@@ -77,11 +78,12 @@ impl From<RawEnvelope> for ParsedEnvelope {
 
  let duplicate_field = fields.iter().any(|(_key, values)| values.len() > 1);
 
+ let content_encoding = remove_field(&mut fields, &CONTENT_ENCODING_TAG);
  let content_type = remove_field(&mut fields, &CONTENT_TYPE_TAG);
+ let metadata = remove_and_concatenate_field(&mut fields, &METADATA_TAG);
+ let metaprotocol = remove_field(&mut fields, &METAPROTOCOL_TAG);
  let parent = remove_field(&mut fields, &PARENT_TAG);
  let pointer = remove_field(&mut fields, &POINTER_TAG);
- let metaprotocol = remove_field(&mut fields, &METAPROTOCOL_TAG);
- let metadata = remove_and_concatenate_field(&mut fields, &METADATA_TAG);
 
  let unrecognized_even_field = fields
  .keys()
@@ -96,14 +98,15 @@ impl From<RawEnvelope> for ParsedEnvelope {
  .cloned()
  .collect()
  }),
+ content_encoding,
  content_type,
- parent,
- pointer,
- unrecognized_even_field,
  duplicate_field,
  incomplete_field,
- metaprotocol,
  metadata,
+ metaprotocol,
+ parent,
+ pointer,
+ unrecognized_even_field,
  },
  input: envelope.input,
  offset: envelope.offset,
@@ -394,13 +397,35 @@ mod tests {
  }
 
  #[test]
- fn with_unknown_tag() {
+ fn with_content_encoding() {
  assert_eq!(
  parse(&[envelope(&[
  b"ord",
  &[1],
  b"text/plain;charset=utf-8",
  &[9],
+ b"br",
+ &[],
+ b"ord",
+ ])]),
+ vec![ParsedEnvelope {
+ payload: Inscription {
+ content_encoding: Some("br".as_bytes().to_vec()),
+ ..inscription("text/plain;charset=utf-8", "ord")
+ },
+ ..Default::default()
+ }]
+ );
+ }
+
+ #[test]
+ fn with_unknown_tag() {
+ assert_eq!(
+ parse(&[envelope(&[
+ b"ord",
+ &[1],
+ b"text/plain;charset=utf-8",
+ &[11],
  b"bar",
  &[],
  b"ord",
@@ -730,7 +755,7 @@ mod tests {
  #[test]
  fn unknown_odd_fields_are_ignored() {
  assert_eq!(
- parse(&[envelope(&[b"ord", &[9], &[0]])]),
+ parse(&[envelope(&[b"ord", &[11], &[0]])]),
  vec![ParsedEnvelope {
  payload: Inscription::default(),
  ..Default::default()

src/inscription.rs

@@ -1,19 +1,23 @@
 use {
  super::*,
+ anyhow::ensure,
  bitcoin::{
  blockdata::{
  opcodes,
  script::{self, PushBytesBuf},
  },
  ScriptBuf,
  },
- io::Cursor,
+ brotli::enc::{writer::CompressorWriter, BrotliEncoderParams},
+ http::header::HeaderValue,
+ io::{Cursor, Read, Write},
  std::str,
 };
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Eq, Default)]
 pub struct Inscription {
  pub body: Option<Vec<u8>>,
+ pub content_encoding: Option<Vec<u8>>,
  pub content_type: Option<Vec<u8>>,
  pub duplicate_field: bool,
  pub incomplete_field: bool,
@@ -41,23 +45,61 @@ impl Inscription {
  pointer: Option<u64>,
  metaprotocol: Option<String>,
  metadata: Option<Vec<u8>>,
+ compress: bool,
  ) -> Result<Self, Error> {
  let path = path.as_ref();
 
  let body = fs::read(path).with_context(|| format!("io error reading {}", path.display()))?;
 
+ let (content_type, compression_mode) = Media::content_type_for_path(path)?;
+
+ let (body, content_encoding) = if compress {
+ let mut compressed = Vec::new();
+
+ {
+ CompressorWriter::with_params(
+ &mut compressed,
+ body.len(),
+ &BrotliEncoderParams {
+ lgblock: 24,
+ lgwin: 24,
+ mode: compression_mode,
+ quality: 11,
+ size_hint: body.len(),
+ ..Default::default()
+ },
+ )
+ .write_all(&body)?;
+
+ let mut decompressor = brotli::Decompressor::new(compressed.as_slice(), compressed.len());
+
+ let mut decompressed = Vec::new();
+
+ decompressor.read_to_end(&mut decompressed)?;
+
+ ensure!(decompressed == body, "decompression roundtrip failed");
+ }
+
+ if compressed.len() < body.len() {
+ (compressed, Some("br".as_bytes().to_vec()))
+ } else {
+ (body, None)
+ }
+ } else {
+ (body, None)
+ };
+
  if let Some(limit) = chain.inscription_content_size_limit() {
  let len = body.len();
  if len > limit {
  bail!("content size of {len} bytes exceeds {limit} byte limit for {chain} inscriptions");
  }
  }
 
- let content_type = Media::content_type_for_path(path)?;
-
  Ok(Self {
  body: Some(body),
  content_type: Some(content_type.into()),
+ content_encoding,
  metadata,
  metaprotocol: metaprotocol.map(|metaprotocol| metaprotocol.into_bytes()),
  parent: parent.map(|id| id.parent_value()),
@@ -91,6 +133,12 @@ impl Inscription {
  .push_slice(PushBytesBuf::try_from(content_type).unwrap());
  }
 
+ if let Some(content_encoding) = self.content_encoding.clone() {
+ builder = builder
+ .push_slice(envelope::CONTENT_ENCODING_TAG)
+ .push_slice(PushBytesBuf::try_from(content_encoding).unwrap());
+ }
+
  if let Some(protocol) = self.metaprotocol.clone() {
  builder = builder
  .push_slice(envelope::METAPROTOCOL_TAG)
@@ -177,6 +225,10 @@ impl Inscription {
  str::from_utf8(self.content_type.as_ref()?).ok()
  }
 
+ pub(crate) fn content_encoding(&self) -> Option<HeaderValue> {
+ HeaderValue::from_str(str::from_utf8(self.content_encoding.as_ref()?).unwrap_or_default()).ok()
+ }
+
  pub(crate) fn metadata(&self) -> Option<Value> {
  ciborium::from_reader(Cursor::new(self.metadata.as_ref()?)).ok()
  }
@@ -692,22 +744,46 @@ mod tests {
  write!(file, "foo").unwrap();
 
  let inscription =
- Inscription::from_file(Chain::Mainnet, file.path(), None, None, None, None).unwrap();
+ Inscription::from_file(Chain::Mainnet, file.path(), None, None, None, None, false).unwrap();
 
  assert_eq!(inscription.pointer, None);
 
- let inscription =
- Inscription::from_file(Chain::Mainnet, file.path(), None, Some(0), None, None).unwrap();
+ let inscription = Inscription::from_file(
+ Chain::Mainnet,
+ file.path(),
+ None,
+ Some(0),
+ None,
+ None,
+ false,
+ )
+ .unwrap();
 
  assert_eq!(inscription.pointer, Some(Vec::new()));
 
- let inscription =
- Inscription::from_file(Chain::Mainnet, file.path(), None, Some(1), None, None).unwrap();
+ let inscription = Inscription::from_file(
+ Chain::Mainnet,
+ file.path(),
+ None,
+ Some(1),
+ None,
+ None,
+ false,
+ )
+ .unwrap();
 
  assert_eq!(inscription.pointer, Some(vec![1]));
 
- let inscription =
- Inscription::from_file(Chain::Mainnet, file.path(), None, Some(256), None, None).unwrap();
+ let inscription = Inscription::from_file(
+ Chain::Mainnet,
+ file.path(),
+ None,
+ Some(256),
+ None,
+ None,
+ false,
+ )
+ .unwrap();
 
  assert_eq!(inscription.pointer, Some(vec![0, 1]));
  }