Optionally respect .gitignore in imdl torrent create

Add a '--ignore' flag that, when passed, causes `imdl torretn create` to
skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and
`git config --get core.excludesFile`.

Also switches from the `walkdir` crate to the `ignore` crate, which uses
`walkdir` internally, and which handles `.gitignore` and `.ignore`
files.

This changes the behavior of `-include-hidden` on MacOS to no longer
skip entries with the hidden attribute set, due to `ignore` not exposing
`walkdir`'s filter functionality.

A PR[0] is pending to add filtering to `ignore`, so hopefully this
functionality can be re-implemented soon.

[0] https://github.com/BurntSushi/ripgrep/pull/1557

type: added
fixes:
- https://github.com/casey/intermodal/issues/378

CHANGELOG.md

@@ -2,9 +2,10 @@ Changelog
 =========
 
 
-UNRELEASED - 2020-04-20
+UNRELEASED - 2020-04-21
 -----------------------
-- :books: [`xxxxxxxxxxxx`](https://github.com/casey/intermodal/commits/master) Improve FAQ template - _Casey Rodarmor <casey@rodarmor.com>_
+- :sparkles: [`xxxxxxxxxxxx`](https://github.com/casey/intermodal/commits/master) Optionally respect `.gitignore` in `imdl torrent create` - Fixes [#378](https://github.com/casey/intermodal/issues/378) - _Celeo <celeodor@gmail.com>_
+- :books: [`9f480624616b`](https://github.com/casey/intermodal/commit/9f480624616b77995befec722effda22cc2d06ad) Improve FAQ template - _Casey Rodarmor <casey@rodarmor.com>_
 - :wrench: [`1380290eb8e2`](https://github.com/casey/intermodal/commit/1380290eb8e222605f368bc8346a1e63c83d9af7) Make `publish-check` recipe stricter - _Casey Rodarmor <casey@rodarmor.com>_
 
 

Cargo.lock

@@ -240,6 +240,27 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
+dependencies = [
+ "crossbeam-utils",
+ "maybe-uninit",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "lazy_static",
+]
+
 [[package]]
 name = "ctor"
 version = "0.1.13"
@@ -445,6 +466,25 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "ignore"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddf60d063dbe6b75388eec66cfc07781167ae3d34a09e0c433e6c5de0511f7fb"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "globset",
+ "lazy_static",
+ "log",
+ "memchr",
+ "regex",
+ "same-file",
+ "thread_local",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "imdl"
 version = "0.1.6"
@@ -456,6 +496,7 @@ dependencies = [
  "claim",
  "console",
  "globset",
+ "ignore",
  "imdl-indicatif",
  "lazy_static",
  "libc",
@@ -480,7 +521,6 @@ dependencies = [
  "temptree",
  "unicode-width",
  "url",
- "walkdir",
 ]
 
 [[package]]

Cargo.toml

@@ -21,6 +21,7 @@ globset           = "0.4.0"
 lazy_static       = "1.4.0"
 libc              = "0.2.0"
 log               = "0.4.8"
+ignore            = "0.4.14"
 md5               = "0.7.0"
 open              = "1.4.0"
 pretty_assertions = "0.6.0"
@@ -38,7 +39,6 @@ syn               = "1.0.14"
 tempfile          = "3.0.0"
 unicode-width     = "0.1.0"
 url               = "2.1.1"
-walkdir           = "2.1.0"
 
 [dependencies.bendy]
 version  = "0.3.0"

book/src/commands/imdl-torrent-create.md

@@ -14,6 +14,9 @@ FLAGS:
     -f, --force               Overwrite the destination `.torrent` file, if it
                               exists.
         --help                Print help message.
+        --ignore              Skip files listed in `.gitignore`, `.ignore`,
+                              `.git/info/exclude`, and `git config --get
+                              core.excludesFile`.
     -h, --include-hidden      Include hidden files that would otherwise be
                               skipped, such as files that start with a `.`, and
                               files hidden by file attributes on macOS and

completions/_imdl

@@ -127,6 +127,7 @@ Sort in ascending order by size, break ties in descending path order:
 '--private[Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.]' \
 '-S[Display information about created torrent file.]' \
 '--show[Display information about created torrent file.]' \
+'--ignore[Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.]' \
 '--help[Print help message.]' \
 '-V[Print version number.]' \
 '--version[Print version number.]' \

completions/_imdl.ps1

@@ -125,6 +125,7 @@ Sort in ascending order by size, break ties in descending path order:
             [CompletionResult]::new('--private', 'private', [CompletionResultType]::ParameterName, 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.')
             [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Display information about created torrent file.')
             [CompletionResult]::new('--show', 'show', [CompletionResultType]::ParameterName, 'Display information about created torrent file.')
+            [CompletionResult]::new('--ignore', 'ignore', [CompletionResultType]::ParameterName, 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help message.')
             [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Print version number.')
             [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Print version number.')

completions/imdl.bash

@@ -135,7 +135,7 @@ _imdl() {
             return 0
             ;;
         imdl__torrent__create)
-            opts=" -n -F -f -h -j -M -O -P -S -V -a -A -t -c -g -i -N -o -p -s  --dry-run --follow-symlinks --force --include-hidden --include-junk --link --md5 --no-created-by --no-creation-date --open --private --show --help --version --announce --allow --announce-tier --comment --node --glob --input --name --sort-by --output --peer --piece-length --source  "
+            opts=" -n -F -f -h -j -M -O -P -S -V -a -A -t -c -g -i -N -o -p -s  --dry-run --follow-symlinks --force --include-hidden --include-junk --link --md5 --no-created-by --no-creation-date --open --private --show --ignore --help --version --announce --allow --announce-tier --comment --node --glob --input --name --sort-by --output --peer --piece-length --source  "
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

completions/imdl.elvish

@@ -118,6 +118,7 @@ Sort in ascending order by size, break ties in descending path order:
             cand --private 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.'
             cand -S 'Display information about created torrent file.'
             cand --show 'Display information about created torrent file.'
+            cand --ignore 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.'
             cand --help 'Print help message.'
             cand -V 'Print version number.'
             cand --version 'Print version number.'

completions/imdl.fish

@@ -66,6 +66,7 @@ complete -c imdl -n "__fish_seen_subcommand_from create" -l no-creation-date -d
 complete -c imdl -n "__fish_seen_subcommand_from create" -s O -l open -d 'Open `.torrent` file after creation. Uses `xdg-open`, `gnome-open`, or `kde-open` on Linux; `open` on macOS; and `cmd /C start` on Windows'
 complete -c imdl -n "__fish_seen_subcommand_from create" -s P -l private -d 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.'
 complete -c imdl -n "__fish_seen_subcommand_from create" -s S -l show -d 'Display information about created torrent file.'
+complete -c imdl -n "__fish_seen_subcommand_from create" -l ignore -d 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.'
 complete -c imdl -n "__fish_seen_subcommand_from create" -l help -d 'Print help message.'
 complete -c imdl -n "__fish_seen_subcommand_from create" -s V -l version -d 'Print version number.'
 complete -c imdl -n "__fish_seen_subcommand_from link" -s i -l input -d 'Generate magnet link from metainfo at `PATH`. If `PATH` is `-`, read metainfo from standard input.'

man/imdl-torrent-create.1

@@ -21,6 +21,10 @@ Overwrite the destination `.torrent` file, if it exists.
 \fB\-\-help\fR
 Print help message.
 .TP
+\fB\-\-ignore\fR
+Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config \fB\-\-get\fR
+core.excludesFile`.
+.TP
 \fB\-h\fR, \fB\-\-include\-hidden\fR
 Include hidden files that would otherwise be skipped, such as files that start with a `.`,
 and files hidden by file attributes on macOS and Windows.

src/common.rs

@@ -27,6 +27,7 @@ pub(crate) use std::{
 pub(crate) use bendy::{decoding::FromBencode, encoding::ToBencode, value::Value};
 pub(crate) use chrono::{TimeZone, Utc};
 pub(crate) use globset::{Glob, GlobMatcher};
+pub(crate) use ignore::WalkBuilder;
 pub(crate) use indicatif::{ProgressBar, ProgressStyle};
 pub(crate) use libc::EXIT_FAILURE;
 pub(crate) use regex::{Regex, RegexSet};
@@ -44,7 +45,6 @@ pub(crate) use strum::{IntoEnumIterator, VariantNames};
 pub(crate) use strum_macros::{EnumIter, EnumString, EnumVariantNames, IntoStaticStr};
 pub(crate) use unicode_width::UnicodeWidthStr;
 pub(crate) use url::{Host, Url};
-pub(crate) use walkdir::WalkDir;
 
 // logging functions
 #[allow(unused_imports)]

src/error.rs

@@ -26,6 +26,8 @@ pub(crate) enum Error {
   FileOrderUnknown { text: String },
   #[snafu(display("I/O error at `{}`: {}", path.display(), source))]
   Filesystem { source: io::Error, path: PathBuf },
+  #[snafu(display("Error searching for files: {}", source))]
+  FileSearch { source: ignore::Error },
   #[snafu(display("Invalid glob: {}", source))]
   GlobParse { source: globset::Error },
   #[snafu(display("Failed to serialize torrent info dictionary: {}", source))]
@@ -171,17 +173,10 @@ impl From<SystemTimeError> for Error {
   }
 }
 
-impl From<walkdir::Error> for Error {
+impl From<ignore::Error> for Error {
-  fn from(walkdir_error: walkdir::Error) -> Self {
+  fn from(ignore_error: ignore::Error) -> Self {
-    let path = walkdir_error
+    Self::FileSearch {
-      .path()
+      source: ignore_error,
-      .invariant_unwrap("Walkdir errors always have path")
-      .to_owned();
-
-    if let Some(source) = walkdir_error.into_io_error() {
-      Self::Filesystem { source, path }
-    } else {
-      Self::internal("Encountered walkdir error without source")
     }
   }
 }

src/subcommand/torrent/create.rs

@@ -240,6 +240,12 @@ Sort in ascending order by size, break ties in descending path order:
             download and upload statistics to multiple trackers."
   )]
   source: Option<String>,
+  #[structopt(
+    long = "ignore",
+    help = "Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config \
+            --get core.excludesFile`."
+  )]
+  ignore: bool,
 }
 
 impl Create {
@@ -1679,12 +1685,6 @@ Content Size  9 bytes
         .arg(env.resolve("foo/hidden")?)
         .status()
         .unwrap();
-    } else if cfg!(target_os = "macos") {
-      Command::new("chflags")
-        .arg("hidden")
-        .arg(env.resolve("foo/hidden")?)
-        .status()
-        .unwrap();
     } else {
       fs::remove_file(env.resolve("foo/hidden")?).unwrap();
     }
@@ -1905,6 +1905,7 @@ Content Size  9 bytes
     );
     assert_eq!(metainfo.info.pieces, PieceList::new());
   }
+
   #[test]
   fn skip_hidden_attribute_dir_contents() -> Result<()> {
     let mut env = test_env! {
@@ -1935,17 +1936,6 @@ Content Size  9 bytes
         .unwrap();
     }
 
-    #[cfg(target_os = "macos")]
-    {
-      env.write("foo/bar/baz", "baz");
-      let path = env.resolve("foo/bar")?;
-      Command::new("chflags")
-        .arg("hidden")
-        .arg(&path)
-        .status()
-        .unwrap();
-    }
-
     env.assert_ok();
     let metainfo = env.load_metainfo("foo.torrent");
     assert_matches!(
@@ -2181,6 +2171,88 @@ Content Size  9 bytes
     assert_eq!(metainfo.info.pieces, PieceList::from_pieces(&["a"]));
   }
 
+  #[test]
+  fn ignore_files_in_gitignore() {
+    let mut env = test_env! {
+      args: [
+        "torrent",
+        "create",
+        "--input",
+        "foo",
+        "--ignore",
+      ],
+      tree: {
+        foo: {
+          ".gitignore": "a",
+          a: "a",
+          b: "b",
+        },
+      }
+    };
+    env.assert_ok();
+    let metainfo = env.load_metainfo("foo.torrent");
+    assert_matches!(
+      metainfo.info.mode,
+      Mode::Multiple { files } if files.len() == 1
+    );
+  }
+
+  #[test]
+  fn ignore_files_in_ignore() {
+    let mut env = test_env! {
+      args: [
+        "torrent",
+        "create",
+        "--input",
+        "foo",
+        "--ignore",
+      ],
+      tree: {
+        foo: {
+          ".ignore": "a",
+          a: "a",
+          b: "b",
+        },
+      }
+    };
+    env.assert_ok();
+    let metainfo = env.load_metainfo("foo.torrent");
+    assert_matches!(
+      metainfo.info.mode,
+      Mode::Multiple { files } if files.len() == 1
+    );
+  }
+
+  #[test]
+  fn ignore_files_in_git_exclude() {
+    let mut env = test_env! {
+      args: [
+        "torrent",
+        "create",
+        "--input",
+        "foo",
+        "--ignore",
+      ],
+      tree: {
+        foo: {
+          ".git": {
+            info: {
+              exclude: "a",
+            },
+          },
+          a: "a",
+          b: "b",
+        },
+      }
+    };
+    env.assert_ok();
+    let metainfo = env.load_metainfo("foo.torrent");
+    assert_matches!(
+      metainfo.info.mode,
+      Mode::Multiple { files } if files.len() == 1
+    );
+  }
+
   #[test]
   fn nodes_default() {
     let mut env = test_env! {

src/subcommand/torrent/create/create_content.rs

@@ -27,6 +27,7 @@ impl CreateContent {
         let files = Walker::new(&env.resolve(path)?)
           .include_junk(create.include_junk)
           .include_hidden(create.include_hidden)
+          .ignore(create.ignore)
           .follow_symlinks(create.follow_symlinks)
           .sort_by(create.sort_by.clone())
           .globs(&create.globs)?

src/subcommand/torrent/stats.rs

@@ -53,7 +53,12 @@ impl Stats {
 
     let mut extractor = Extractor::new(self.print, &self.extract_patterns);
 
-    for result in WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name())) {
+    for result in WalkBuilder::new(path)
+      .standard_filters(false)
+      .hidden(true)
+      .sort_by_file_name(|a, b| a.cmp(b))
+      .build()
+    {
       if extractor.torrents >= self.limit.unwrap_or(u64::max_value()) {
         break;
       }

src/walker.rs

@@ -12,6 +12,7 @@ pub(crate) struct Walker {
   follow_symlinks: bool,
   include_hidden: bool,
   include_junk: bool,
+  ignore: bool,
   sort_by: Vec<SortSpec>,
   patterns: Vec<Pattern>,
   root: PathBuf,
@@ -24,6 +25,7 @@ impl Walker {
       follow_symlinks: false,
       include_hidden: false,
       include_junk: false,
+      ignore: false,
       sort_by: Vec::new(),
       patterns: Vec::new(),
       root: root.to_owned(),
@@ -45,6 +47,10 @@ impl Walker {
     }
   }
 
+  pub(crate) fn ignore(self, ignore: bool) -> Self {
+    Self { ignore, ..self }
+  }
+
   pub(crate) fn sort_by(self, sort_by: Vec<SortSpec>) -> Self {
     Self { sort_by, ..self }
   }
@@ -94,7 +100,17 @@ impl Walker {
       return Ok(Files::file(self.root, Bytes::from(root_metadata.len())));
     }
 
-    let filter = |entry: &walkdir::DirEntry| {
+    let mut file_infos = Vec::new();
+    let mut total_size = 0;
+
+    let mut walk_builder = WalkBuilder::new(&self.root);
+    walk_builder
+      .follow_links(self.follow_symlinks)
+      .standard_filters(self.ignore)
+      .require_git(false)
+      .hidden(!self.include_hidden);
+    for result in walk_builder.build() {
+      let entry = result?;
       let path = entry.path();
 
       if let Some(s) = &self.spinner {
@@ -103,32 +119,6 @@ impl Walker {
         s.tick();
       }
 
-      let file_name = entry.file_name();
-
-      if !self.include_hidden && file_name.to_string_lossy().starts_with('.') {
-        return false;
-      }
-
-      let hidden = Platform::hidden(path).unwrap_or(true);
-
-      if !self.include_hidden && hidden {
-        return false;
-      }
-
-      true
-    };
-
-    let mut file_infos = Vec::new();
-    let mut total_size = 0;
-    for result in WalkDir::new(&self.root)
-      .follow_links(self.follow_symlinks)
-      .into_iter()
-      .filter_entry(filter)
-    {
-      let entry = result?;
-
-      let path = entry.path();
-
       let metadata = entry.metadata()?;
 
       if !metadata.is_file() {