Initial commit of the data crate

The `data` crate is intended to be used as the manifest serialization
and deserialization format.

This blog post describes the motivation and goals for this crate:

    https://rodarmor.com/blog/data

type: added

Cargo.lock

@@ -199,7 +199,7 @@ dependencies = [
  "ansi_term 0.11.0",
  "atty",
  "bitflags",
- "strsim",
+ "strsim 0.8.0",
  "term_size",
  "textwrap",
  "unicode-width",
@@ -265,6 +265,58 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "darling"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim 0.9.3",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "data"
+version = "0.1.0"
+dependencies = [
+ "data-macros",
+]
+
+[[package]]
+name = "data-macros"
+version = "0.0.0"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "demo"
 version = "0.0.0"
@@ -458,6 +510,12 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
 [[package]]
 name = "idna"
 version = "0.2.0"
@@ -801,9 +859,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.10"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3"
+checksum = "8872cf6f48eee44265156c111456a700ab3483686b3f96df4cf5481c89157319"
 dependencies = [
  "unicode-xid",
 ]
@@ -816,9 +874,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quote"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f"
+checksum = "4c1f4b0efa5fc5e8ceb705136bfee52cfdb6a4e3509f770b478cd6ed434232a7"
 dependencies = [
  "proc-macro2",
 ]
@@ -1039,6 +1097,12 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
+[[package]]
+name = "strsim"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
+
 [[package]]
 name = "structopt"
 version = "0.3.14"
@@ -1083,9 +1147,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03"
+checksum = "410a7488c0a728c7ceb4ad59b9567eb4053d02e8cc7f5c0e0eeeb39518369213"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -71,4 +71,10 @@ members = [
 
   # run commands for demo animation
   "bin/demo",
+
+  # data description language
+  "crates/data",
+
+  # data description language proc macros
+  "crates/data-macros",
 ]

crates/data-macros/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name    = "data-macros"
+version = "0.0.0"
+authors = ["Casey Rodarmor <casey@rodarmor.com>"]
+edition = "2018"
+publish = false
+
+[lib]
+proc-macro = true
+
+[dependencies]
+darling     = "0.10.2"
+proc-macro2 = "1.0.12"
+quote       = "1.0.4"
+syn         = "1.0.18"

crates/data-macros/src/attribute.rs

@@ -0,0 +1,28 @@
+use crate::common::*;
+
+pub(crate) trait Attribute: FromMeta {
+  type Item: syn::parse::Parse;
+
+  fn attribute(
+    attr: proc_macro::TokenStream,
+    item: proc_macro::TokenStream,
+  ) -> proc_macro::TokenStream {
+    Self::inner(attr.into(), item.into()).tokens().into()
+  }
+
+  fn inner(attr: TokenStream, item: TokenStream) -> Result<TokenStream, Error> {
+    let args = Punctuated::<NestedMeta, token::Comma>::parse_terminated
+      .parse2(attr)?
+      .into_iter()
+      .collect();
+
+    Self::innermost(args, item)
+  }
+
+  fn innermost(args: Vec<NestedMeta>, item: TokenStream) -> Result<TokenStream, Error> {
+    let item = syn::parse2::<Self::Item>(item)?;
+    Self::from_list(&args)?.expand(item)
+  }
+
+  fn expand(&self, item: Self::Item) -> Result<TokenStream, Error>;
+}

crates/data-macros/src/common.rs

@@ -0,0 +1,8 @@
+pub(crate) use darling::FromMeta;
+pub(crate) use proc_macro2::TokenStream;
+pub(crate) use quote::ToTokens;
+pub(crate) use syn::{parse::Parser, punctuated::Punctuated, token, ItemTrait, NestedMeta};
+
+pub(crate) use crate::{attribute::Attribute, tokens::Tokens};
+
+pub(crate) use crate::{error::Error, table::Table};

crates/data-macros/src/error.rs

@@ -0,0 +1,28 @@
+use crate::common::*;
+
+#[derive(Debug)]
+pub(crate) enum Error {
+  Parse(darling::Error),
+  Syn(syn::Error),
+}
+
+impl From<darling::Error> for Error {
+  fn from(error: darling::Error) -> Error {
+    Error::Parse(error)
+  }
+}
+
+impl From<syn::Error> for Error {
+  fn from(error: syn::Error) -> Error {
+    Error::Syn(error)
+  }
+}
+
+impl Tokens for Error {
+  fn tokens(self) -> TokenStream {
+    match self {
+      Error::Parse(error) => error.write_errors(),
+      Error::Syn(error) => error.to_compile_error(),
+    }
+  }
+}

crates/data-macros/src/lib.rs

@@ -0,0 +1,19 @@
+use crate::common::*;
+
+#[cfg(test)]
+#[macro_use]
+mod test;
+
+mod attribute;
+mod common;
+mod error;
+mod table;
+mod tokens;
+
+#[proc_macro_attribute]
+pub fn table(
+  attr: proc_macro::TokenStream,
+  item: proc_macro::TokenStream,
+) -> proc_macro::TokenStream {
+  Table::attribute(attr, item)
+}

crates/data-macros/src/table.rs

@@ -0,0 +1,24 @@
+use crate::common::*;
+
+#[derive(FromMeta)]
+pub(crate) struct Table {}
+
+impl Attribute for Table {
+  type Item = ItemTrait;
+
+  fn expand(&self, item: Self::Item) -> Result<TokenStream, Error> {
+    Ok(item.into_token_stream())
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  #[test]
+  fn empty() {
+    assert_attribute_expansion_eq!(
+      #[data::table]
+      trait Foo {},
+      trait Foo {}
+    );
+  }
+}

crates/data-macros/src/test.rs

@@ -0,0 +1,66 @@
+use crate::common::*;
+
+use syn::{Meta, MetaNameValue, NestedMeta};
+
+macro_rules! assert_attribute_expansion_eq {
+  {
+    #[$meta:meta]
+    $item:item,
+    $($expansion:tt)*
+  } => {
+    {
+      let have = expand_attribute!(#[$meta] $item).unwrap().to_string();
+      let want = quote::quote!($($expansion)*).to_string();
+      assert_eq!(have, want);
+    }
+  }
+}
+
+macro_rules! expand_attribute {
+  {
+    #[$meta:meta]
+    $item:item
+  } => {
+    {
+      let meta = quote::quote!($meta);
+      let item = quote::quote!($item);
+      $crate::test::expand_attribute(meta, item)
+    }
+  }
+}
+
+pub(crate) fn expand_attribute(meta: TokenStream, item: TokenStream) -> Result<TokenStream, Error> {
+  let meta = syn::parse2::<Meta>(meta).unwrap();
+
+  let (path, args) = split_meta(meta);
+
+  match path.as_str() {
+    "data::table" => Table::innermost(args, item),
+    _ => panic!("attribute `{}` unknown", path),
+  }
+}
+
+fn split_meta(meta: Meta) -> (String, Vec<NestedMeta>) {
+  fn text(path: syn::Path) -> String {
+    let mut text = String::new();
+
+    if let Some(_) = path.leading_colon {
+      text.push_str("::");
+    }
+
+    for (i, segment) in path.segments.iter().enumerate() {
+      if i > 0 {
+        text.push_str("::");
+      }
+      text.push_str(&segment.ident.to_string());
+    }
+
+    text
+  }
+
+  match meta {
+    Meta::Path(path) => (text(path), Vec::new()),
+    Meta::List(syn::MetaList { path, nested, .. }) => (text(path), nested.into_iter().collect()),
+    Meta::NameValue(MetaNameValue { path, lit, .. }) => (text(path), vec![NestedMeta::Lit(lit)]),
+  }
+}

crates/data-macros/src/tokens.rs

@@ -0,0 +1,20 @@
+use crate::common::*;
+
+pub(crate) trait Tokens {
+  fn tokens(self) -> TokenStream;
+}
+
+impl Tokens for TokenStream {
+  fn tokens(self) -> TokenStream {
+    self
+  }
+}
+
+impl<T: Tokens, E: Tokens> Tokens for Result<T, E> {
+  fn tokens(self) -> TokenStream {
+    match self {
+      Ok(t) => t.tokens(),
+      Err(e) => e.tokens(),
+    }
+  }
+}

crates/data/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name    = "data"
+version = "0.1.0"
+authors = ["Casey Rodarmor <casey@rodarmor.com>"]
+edition = "2018"
+
+[dependencies.data-macros]
+path = "../data-macros"

crates/data/src/buffer.rs

@@ -0,0 +1,36 @@
+use crate::common::*;
+
+pub(crate) struct Buffer {
+  layout: Layout,
+  alloc: *mut u8,
+}
+
+impl Buffer {
+  pub(crate) fn new<'a, T: Data<'a>>(contents: &[u8]) -> Buffer {
+    let layout = Layout::from_size_align(T::FIXED_SIZE, T::ALIGNMENT).unwrap();
+    let alloc = unsafe { alloc::alloc::alloc_zeroed(layout) };
+    let mut buffer = Buffer { layout, alloc };
+    buffer.deref_mut().copy_from_slice(contents);
+    buffer
+  }
+}
+
+impl Drop for Buffer {
+  fn drop(&mut self) {
+    unsafe { alloc::alloc::dealloc(self.alloc, self.layout) };
+  }
+}
+
+impl Deref for Buffer {
+  type Target = [u8];
+
+  fn deref(&self) -> &Self::Target {
+    unsafe { core::slice::from_raw_parts(self.alloc, self.layout.size()) }
+  }
+}
+
+impl DerefMut for Buffer {
+  fn deref_mut(&mut self) -> &mut Self::Target {
+    unsafe { core::slice::from_raw_parts_mut(self.alloc, self.layout.size()) }
+  }
+}

crates/data/src/common.rs

@@ -0,0 +1,26 @@
+pub(crate) use core::{result, str::Utf8Error};
+
+pub(crate) use crate::error::Error;
+
+pub(crate) use crate::data::Data;
+
+pub(crate) type Result<T, E = Error> = result::Result<T, E>;
+
+#[cfg(test)]
+mod test {
+  pub(crate) extern crate alloc;
+
+  pub(crate) use core::{
+    fmt::Debug,
+    ops::{Deref, DerefMut},
+  };
+
+  pub(crate) use alloc::{alloc::Layout, vec::Vec};
+
+  pub(crate) use crate::buffer::Buffer;
+
+  pub(crate) use crate::data_test::DataTest;
+}
+
+#[cfg(test)]
+pub(crate) use test::*;

crates/data/src/data.rs

@@ -0,0 +1,27 @@
+use crate::common::*;
+
+pub trait Data<'a>: Sized + 'a {
+  const FIXED_SIZE: usize;
+
+  const ALIGNMENT: usize;
+
+  fn load(buffer: &'a [u8]) -> Result<Self>;
+
+  fn store(&self, buffer: &mut [u8]) -> Result<()>;
+
+  fn check(buffer: &'a [u8]) -> Result<()> {
+    let have = buffer.len();
+    let want = Self::FIXED_SIZE;
+    if have < want {
+      return Err(Error::Size { have, want });
+    }
+
+    let alignment = Self::ALIGNMENT;
+    let offset = buffer.as_ptr().align_offset(alignment);
+    if offset != 0 {
+      return Err(Error::Alignment { alignment, offset });
+    }
+
+    Ok(())
+  }
+}

crates/data/src/data_test.rs

@@ -0,0 +1,38 @@
+use crate::common::*;
+
+pub trait DataTest<'a>: Data<'a> + PartialEq + Debug + Default {
+  fn round_trip(self, encoded: &'a [u8]) -> Result<()> {
+    let mut buffer = Vec::<u8>::with_capacity(Self::FIXED_SIZE);
+    buffer.resize(Self::FIXED_SIZE, 0);
+    self.store(&mut buffer)?;
+    assert_eq!(buffer, encoded);
+    let decoded = Self::load(&encoded)?;
+    assert_eq!(self, decoded);
+    Ok(())
+  }
+
+  fn error_test(buffer: &'a mut [u8]) {
+    let want = Self::FIXED_SIZE;
+    let have = Self::FIXED_SIZE - 1;
+    assert_eq!(buffer.len(), want);
+    let short_buffer = &mut buffer[..have];
+    assert_eq!(
+      Self::default().store(short_buffer),
+      Err(Error::Size { have, want })
+    );
+    assert_eq!(Self::load(short_buffer), Err(Error::Size { have, want }));
+  }
+}
+
+impl<'a, T: Data<'a> + PartialEq + Debug + Default> DataTest<'a> for T {}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  #[should_panic]
+  fn round_trip_test() {
+    0u8.round_trip(&[1]).ok();
+  }
+}

crates/data/src/error.rs

@@ -0,0 +1,8 @@
+use crate::common::*;
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+  Alignment { alignment: usize, offset: usize },
+  UnicodeDecode { source: Utf8Error },
+  Size { have: usize, want: usize },
+}

crates/data/src/lib.rs

@@ -0,0 +1,17 @@
+#![no_std]
+
+mod common;
+mod data;
+mod error;
+mod u16;
+mod u8;
+
+#[cfg(test)]
+mod buffer;
+
+#[cfg(test)]
+mod data_test;
+
+pub use data_macros::table;
+
+pub use crate::{data::Data, error::Error};

crates/data/src/u16.rs

@@ -0,0 +1,38 @@
+use crate::common::*;
+
+impl<'a> Data<'a> for u16 {
+  const ALIGNMENT: usize = 2;
+  const FIXED_SIZE: usize = 2;
+
+  fn load(buffer: &'a [u8]) -> Result<Self> {
+    Self::check(buffer)?;
+    Ok(u16::from_le_bytes([buffer[0], buffer[1]]))
+  }
+
+  fn store(&self, buffer: &mut [u8]) -> Result<()> {
+    Self::check(buffer)?;
+    let bytes = self.to_le_bytes();
+    buffer.copy_from_slice(&bytes);
+    Ok(())
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn u16_round_trip() -> Result<()> {
+    0u16.round_trip(&Buffer::new::<u16>(&[0, 0]))?;
+    1u16.round_trip(&Buffer::new::<u16>(&[1, 0]))?;
+    255u16.round_trip(&Buffer::new::<u16>(&[255, 0]))?;
+    256u16.round_trip(&Buffer::new::<u16>(&[0, 1]))?;
+    u16::max_value().round_trip(&Buffer::new::<u16>(&[0xFF, 0xFF]))?;
+    Ok(())
+  }
+
+  #[test]
+  fn u16_errors() {
+    u16::error_test(&mut Buffer::new::<u16>(&[0, 0]));
+  }
+}

crates/data/src/u8.rs

@@ -0,0 +1,35 @@
+use crate::common::*;
+
+impl<'a> Data<'a> for u8 {
+  const ALIGNMENT: usize = 1;
+  const FIXED_SIZE: usize = 1;
+
+  fn load(buffer: &'a [u8]) -> Result<Self> {
+    Self::check(buffer)?;
+    Ok(buffer[0])
+  }
+
+  fn store(&self, buffer: &mut [u8]) -> Result<()> {
+    Self::check(buffer)?;
+    buffer[0] = *self;
+    Ok(())
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn u8_round_trip() -> Result<()> {
+    0u8.round_trip(&[0])?;
+    1u8.round_trip(&[1])?;
+    255u8.round_trip(&[255])?;
+    Ok(())
+  }
+
+  #[test]
+  fn u8_errors() {
+    u8::error_test(&mut Buffer::new::<u8>(&[0]));
+  }
+}

crates/data/tests/manifest.rs

@@ -0,0 +1,48 @@
+#[cfg(aspirational)]
+mod aspirational {
+
+  #[data::structure]
+  struct Manifest {
+    magic_number: [u8; 8], // IMDL + 4 bytes that aren't valid UTF-8
+    version: Version,      // 256.256.256
+  }
+
+  #[data::structure]
+  struct Version {
+    major: u64,
+    minor: u64,
+    patch: u64,
+  }
+
+  #[data::table]
+  struct Manifest;
+
+  #[data::table_impl]
+  impl Manifest {
+    fn files(&self) -> Slice<Hash>;
+
+    fn directory(&self) -> Directory;
+  }
+
+  #[data::structure]
+  struct Directory {
+    records: Slice<Record>,
+  }
+
+  #[data::structure]
+  struct Record {
+    name: &str,
+    node: Node,
+  }
+
+  #[data::enumeration]
+  enum Node {
+    Directory(Directory),
+    File(u64),
+  }
+
+  #[data::structure]
+  struct Hash {
+    bytes: [u8; 32],
+  }
+}