From 9b72873ed13e8f0ae747714545c48c6e37c67dd0 Mon Sep 17 00:00:00 2001 From: Celeo Date: Wed, 15 Apr 2020 20:17:40 -0700 Subject: [PATCH] Optionally respect `.gitignore` in `imdl torrent create` Add a '--ignore' flag that, when passed, causes `imdl torretn create` to skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`. Also switches from the `walkdir` crate to the `ignore` crate, which uses `walkdir` internally, and which handles `.gitignore` and `.ignore` files. This changes the behavior of `-include-hidden` on MacOS to no longer skip entries with the hidden attribute set, due to `ignore` not exposing `walkdir`'s filter functionality. A PR[0] is pending to add filtering to `ignore`, so hopefully this functionality can be re-implemented soon. [0] https://github.com/BurntSushi/ripgrep/pull/1557 type: added fixes: - https://github.com/casey/intermodal/issues/378 --- CHANGELOG.md | 5 +- Cargo.lock | 42 ++++++- Cargo.toml | 2 +- book/src/commands/imdl-torrent-create.md | 3 + completions/_imdl | 1 + completions/_imdl.ps1 | 1 + completions/imdl.bash | 2 +- completions/imdl.elvish | 1 + completions/imdl.fish | 1 + man/imdl-torrent-create.1 | 4 + src/common.rs | 2 +- src/error.rs | 17 +-- src/subcommand/torrent/create.rs | 106 +++++++++++++++--- .../torrent/create/create_content.rs | 1 + src/subcommand/torrent/stats.rs | 7 +- src/walker.rs | 44 +++----- 16 files changed, 177 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c9d3b4..7b43526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,10 @@ Changelog ========= -UNRELEASED - 2020-04-20 +UNRELEASED - 2020-04-21 ----------------------- -- :books: [`xxxxxxxxxxxx`](https://github.com/casey/intermodal/commits/master) Improve FAQ template - _Casey Rodarmor _ +- :sparkles: [`xxxxxxxxxxxx`](https://github.com/casey/intermodal/commits/master) Optionally respect `.gitignore` in `imdl torrent create` - Fixes [#378](https://github.com/casey/intermodal/issues/378) - _Celeo _ +- :books: [`9f480624616b`](https://github.com/casey/intermodal/commit/9f480624616b77995befec722effda22cc2d06ad) Improve FAQ template - _Casey Rodarmor _ - :wrench: [`1380290eb8e2`](https://github.com/casey/intermodal/commit/1380290eb8e222605f368bc8346a1e63c83d9af7) Make `publish-check` recipe stricter - _Casey Rodarmor _ diff --git a/Cargo.lock b/Cargo.lock index 777d7a3..d09eb97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,6 +240,27 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "crossbeam-channel" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" +dependencies = [ + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + [[package]] name = "ctor" version = "0.1.13" @@ -445,6 +466,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ignore" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf60d063dbe6b75388eec66cfc07781167ae3d34a09e0c433e6c5de0511f7fb" +dependencies = [ + "crossbeam-channel", + "crossbeam-utils", + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + [[package]] name = "imdl" version = "0.1.6" @@ -456,6 +496,7 @@ dependencies = [ "claim", "console", "globset", + "ignore", "imdl-indicatif", "lazy_static", "libc", @@ -480,7 +521,6 @@ dependencies = [ "temptree", "unicode-width", "url", - "walkdir", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a492045..a5fa383 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ globset = "0.4.0" lazy_static = "1.4.0" libc = "0.2.0" log = "0.4.8" +ignore = "0.4.14" md5 = "0.7.0" open = "1.4.0" pretty_assertions = "0.6.0" @@ -38,7 +39,6 @@ syn = "1.0.14" tempfile = "3.0.0" unicode-width = "0.1.0" url = "2.1.1" -walkdir = "2.1.0" [dependencies.bendy] version = "0.3.0" diff --git a/book/src/commands/imdl-torrent-create.md b/book/src/commands/imdl-torrent-create.md index b57112e..2336adf 100644 --- a/book/src/commands/imdl-torrent-create.md +++ b/book/src/commands/imdl-torrent-create.md @@ -14,6 +14,9 @@ FLAGS: -f, --force Overwrite the destination `.torrent` file, if it exists. --help Print help message. + --ignore Skip files listed in `.gitignore`, `.ignore`, + `.git/info/exclude`, and `git config --get + core.excludesFile`. -h, --include-hidden Include hidden files that would otherwise be skipped, such as files that start with a `.`, and files hidden by file attributes on macOS and diff --git a/completions/_imdl b/completions/_imdl index e58be28..d79855f 100644 --- a/completions/_imdl +++ b/completions/_imdl @@ -127,6 +127,7 @@ Sort in ascending order by size, break ties in descending path order: '--private[Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.]' \ '-S[Display information about created torrent file.]' \ '--show[Display information about created torrent file.]' \ +'--ignore[Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.]' \ '--help[Print help message.]' \ '-V[Print version number.]' \ '--version[Print version number.]' \ diff --git a/completions/_imdl.ps1 b/completions/_imdl.ps1 index a5567c1..f30751e 100644 --- a/completions/_imdl.ps1 +++ b/completions/_imdl.ps1 @@ -125,6 +125,7 @@ Sort in ascending order by size, break ties in descending path order: [CompletionResult]::new('--private', 'private', [CompletionResultType]::ParameterName, 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.') [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Display information about created torrent file.') [CompletionResult]::new('--show', 'show', [CompletionResultType]::ParameterName, 'Display information about created torrent file.') + [CompletionResult]::new('--ignore', 'ignore', [CompletionResultType]::ParameterName, 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help message.') [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Print version number.') [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Print version number.') diff --git a/completions/imdl.bash b/completions/imdl.bash index 4d77e59..62b8469 100644 --- a/completions/imdl.bash +++ b/completions/imdl.bash @@ -135,7 +135,7 @@ _imdl() { return 0 ;; imdl__torrent__create) - opts=" -n -F -f -h -j -M -O -P -S -V -a -A -t -c -g -i -N -o -p -s --dry-run --follow-symlinks --force --include-hidden --include-junk --link --md5 --no-created-by --no-creation-date --open --private --show --help --version --announce --allow --announce-tier --comment --node --glob --input --name --sort-by --output --peer --piece-length --source " + opts=" -n -F -f -h -j -M -O -P -S -V -a -A -t -c -g -i -N -o -p -s --dry-run --follow-symlinks --force --include-hidden --include-junk --link --md5 --no-created-by --no-creation-date --open --private --show --ignore --help --version --announce --allow --announce-tier --comment --node --glob --input --name --sort-by --output --peer --piece-length --source " if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 diff --git a/completions/imdl.elvish b/completions/imdl.elvish index 30e5e6d..ddc783c 100644 --- a/completions/imdl.elvish +++ b/completions/imdl.elvish @@ -118,6 +118,7 @@ Sort in ascending order by size, break ties in descending path order: cand --private 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.' cand -S 'Display information about created torrent file.' cand --show 'Display information about created torrent file.' + cand --ignore 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.' cand --help 'Print help message.' cand -V 'Print version number.' cand --version 'Print version number.' diff --git a/completions/imdl.fish b/completions/imdl.fish index b8e0a26..71934c5 100644 --- a/completions/imdl.fish +++ b/completions/imdl.fish @@ -66,6 +66,7 @@ complete -c imdl -n "__fish_seen_subcommand_from create" -l no-creation-date -d complete -c imdl -n "__fish_seen_subcommand_from create" -s O -l open -d 'Open `.torrent` file after creation. Uses `xdg-open`, `gnome-open`, or `kde-open` on Linux; `open` on macOS; and `cmd /C start` on Windows' complete -c imdl -n "__fish_seen_subcommand_from create" -s P -l private -d 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.' complete -c imdl -n "__fish_seen_subcommand_from create" -s S -l show -d 'Display information about created torrent file.' +complete -c imdl -n "__fish_seen_subcommand_from create" -l ignore -d 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.' complete -c imdl -n "__fish_seen_subcommand_from create" -l help -d 'Print help message.' complete -c imdl -n "__fish_seen_subcommand_from create" -s V -l version -d 'Print version number.' complete -c imdl -n "__fish_seen_subcommand_from link" -s i -l input -d 'Generate magnet link from metainfo at `PATH`. If `PATH` is `-`, read metainfo from standard input.' diff --git a/man/imdl-torrent-create.1 b/man/imdl-torrent-create.1 index 1949429..1ad0146 100644 --- a/man/imdl-torrent-create.1 +++ b/man/imdl-torrent-create.1 @@ -21,6 +21,10 @@ Overwrite the destination `.torrent` file, if it exists. \fB\-\-help\fR Print help message. .TP +\fB\-\-ignore\fR +Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config \fB\-\-get\fR +core.excludesFile`. +.TP \fB\-h\fR, \fB\-\-include\-hidden\fR Include hidden files that would otherwise be skipped, such as files that start with a `.`, and files hidden by file attributes on macOS and Windows. diff --git a/src/common.rs b/src/common.rs index d868223..1a757c5 100644 --- a/src/common.rs +++ b/src/common.rs @@ -27,6 +27,7 @@ pub(crate) use std::{ pub(crate) use bendy::{decoding::FromBencode, encoding::ToBencode, value::Value}; pub(crate) use chrono::{TimeZone, Utc}; pub(crate) use globset::{Glob, GlobMatcher}; +pub(crate) use ignore::WalkBuilder; pub(crate) use indicatif::{ProgressBar, ProgressStyle}; pub(crate) use libc::EXIT_FAILURE; pub(crate) use regex::{Regex, RegexSet}; @@ -44,7 +45,6 @@ pub(crate) use strum::{IntoEnumIterator, VariantNames}; pub(crate) use strum_macros::{EnumIter, EnumString, EnumVariantNames, IntoStaticStr}; pub(crate) use unicode_width::UnicodeWidthStr; pub(crate) use url::{Host, Url}; -pub(crate) use walkdir::WalkDir; // logging functions #[allow(unused_imports)] diff --git a/src/error.rs b/src/error.rs index 6f9889a..8c9cca3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,8 @@ pub(crate) enum Error { FileOrderUnknown { text: String }, #[snafu(display("I/O error at `{}`: {}", path.display(), source))] Filesystem { source: io::Error, path: PathBuf }, + #[snafu(display("Error searching for files: {}", source))] + FileSearch { source: ignore::Error }, #[snafu(display("Invalid glob: {}", source))] GlobParse { source: globset::Error }, #[snafu(display("Failed to serialize torrent info dictionary: {}", source))] @@ -171,17 +173,10 @@ impl From for Error { } } -impl From for Error { - fn from(walkdir_error: walkdir::Error) -> Self { - let path = walkdir_error - .path() - .invariant_unwrap("Walkdir errors always have path") - .to_owned(); - - if let Some(source) = walkdir_error.into_io_error() { - Self::Filesystem { source, path } - } else { - Self::internal("Encountered walkdir error without source") +impl From for Error { + fn from(ignore_error: ignore::Error) -> Self { + Self::FileSearch { + source: ignore_error, } } } diff --git a/src/subcommand/torrent/create.rs b/src/subcommand/torrent/create.rs index 6907633..dd331ac 100644 --- a/src/subcommand/torrent/create.rs +++ b/src/subcommand/torrent/create.rs @@ -240,6 +240,12 @@ Sort in ascending order by size, break ties in descending path order: download and upload statistics to multiple trackers." )] source: Option, + #[structopt( + long = "ignore", + help = "Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config \ + --get core.excludesFile`." + )] + ignore: bool, } impl Create { @@ -1679,12 +1685,6 @@ Content Size 9 bytes .arg(env.resolve("foo/hidden")?) .status() .unwrap(); - } else if cfg!(target_os = "macos") { - Command::new("chflags") - .arg("hidden") - .arg(env.resolve("foo/hidden")?) - .status() - .unwrap(); } else { fs::remove_file(env.resolve("foo/hidden")?).unwrap(); } @@ -1905,6 +1905,7 @@ Content Size 9 bytes ); assert_eq!(metainfo.info.pieces, PieceList::new()); } + #[test] fn skip_hidden_attribute_dir_contents() -> Result<()> { let mut env = test_env! { @@ -1935,17 +1936,6 @@ Content Size 9 bytes .unwrap(); } - #[cfg(target_os = "macos")] - { - env.write("foo/bar/baz", "baz"); - let path = env.resolve("foo/bar")?; - Command::new("chflags") - .arg("hidden") - .arg(&path) - .status() - .unwrap(); - } - env.assert_ok(); let metainfo = env.load_metainfo("foo.torrent"); assert_matches!( @@ -2181,6 +2171,88 @@ Content Size 9 bytes assert_eq!(metainfo.info.pieces, PieceList::from_pieces(&["a"])); } + #[test] + fn ignore_files_in_gitignore() { + let mut env = test_env! { + args: [ + "torrent", + "create", + "--input", + "foo", + "--ignore", + ], + tree: { + foo: { + ".gitignore": "a", + a: "a", + b: "b", + }, + } + }; + env.assert_ok(); + let metainfo = env.load_metainfo("foo.torrent"); + assert_matches!( + metainfo.info.mode, + Mode::Multiple { files } if files.len() == 1 + ); + } + + #[test] + fn ignore_files_in_ignore() { + let mut env = test_env! { + args: [ + "torrent", + "create", + "--input", + "foo", + "--ignore", + ], + tree: { + foo: { + ".ignore": "a", + a: "a", + b: "b", + }, + } + }; + env.assert_ok(); + let metainfo = env.load_metainfo("foo.torrent"); + assert_matches!( + metainfo.info.mode, + Mode::Multiple { files } if files.len() == 1 + ); + } + + #[test] + fn ignore_files_in_git_exclude() { + let mut env = test_env! { + args: [ + "torrent", + "create", + "--input", + "foo", + "--ignore", + ], + tree: { + foo: { + ".git": { + info: { + exclude: "a", + }, + }, + a: "a", + b: "b", + }, + } + }; + env.assert_ok(); + let metainfo = env.load_metainfo("foo.torrent"); + assert_matches!( + metainfo.info.mode, + Mode::Multiple { files } if files.len() == 1 + ); + } + #[test] fn nodes_default() { let mut env = test_env! { diff --git a/src/subcommand/torrent/create/create_content.rs b/src/subcommand/torrent/create/create_content.rs index 664fda6..00c0d5f 100644 --- a/src/subcommand/torrent/create/create_content.rs +++ b/src/subcommand/torrent/create/create_content.rs @@ -27,6 +27,7 @@ impl CreateContent { let files = Walker::new(&env.resolve(path)?) .include_junk(create.include_junk) .include_hidden(create.include_hidden) + .ignore(create.ignore) .follow_symlinks(create.follow_symlinks) .sort_by(create.sort_by.clone()) .globs(&create.globs)? diff --git a/src/subcommand/torrent/stats.rs b/src/subcommand/torrent/stats.rs index 59e0041..0dca0e5 100644 --- a/src/subcommand/torrent/stats.rs +++ b/src/subcommand/torrent/stats.rs @@ -53,7 +53,12 @@ impl Stats { let mut extractor = Extractor::new(self.print, &self.extract_patterns); - for result in WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name())) { + for result in WalkBuilder::new(path) + .standard_filters(false) + .hidden(true) + .sort_by_file_name(|a, b| a.cmp(b)) + .build() + { if extractor.torrents >= self.limit.unwrap_or(u64::max_value()) { break; } diff --git a/src/walker.rs b/src/walker.rs index b6d5c6d..f00f974 100644 --- a/src/walker.rs +++ b/src/walker.rs @@ -12,6 +12,7 @@ pub(crate) struct Walker { follow_symlinks: bool, include_hidden: bool, include_junk: bool, + ignore: bool, sort_by: Vec, patterns: Vec, root: PathBuf, @@ -24,6 +25,7 @@ impl Walker { follow_symlinks: false, include_hidden: false, include_junk: false, + ignore: false, sort_by: Vec::new(), patterns: Vec::new(), root: root.to_owned(), @@ -45,6 +47,10 @@ impl Walker { } } + pub(crate) fn ignore(self, ignore: bool) -> Self { + Self { ignore, ..self } + } + pub(crate) fn sort_by(self, sort_by: Vec) -> Self { Self { sort_by, ..self } } @@ -94,7 +100,17 @@ impl Walker { return Ok(Files::file(self.root, Bytes::from(root_metadata.len()))); } - let filter = |entry: &walkdir::DirEntry| { + let mut file_infos = Vec::new(); + let mut total_size = 0; + + let mut walk_builder = WalkBuilder::new(&self.root); + walk_builder + .follow_links(self.follow_symlinks) + .standard_filters(self.ignore) + .require_git(false) + .hidden(!self.include_hidden); + for result in walk_builder.build() { + let entry = result?; let path = entry.path(); if let Some(s) = &self.spinner { @@ -103,32 +119,6 @@ impl Walker { s.tick(); } - let file_name = entry.file_name(); - - if !self.include_hidden && file_name.to_string_lossy().starts_with('.') { - return false; - } - - let hidden = Platform::hidden(path).unwrap_or(true); - - if !self.include_hidden && hidden { - return false; - } - - true - }; - - let mut file_infos = Vec::new(); - let mut total_size = 0; - for result in WalkDir::new(&self.root) - .follow_links(self.follow_symlinks) - .into_iter() - .filter_entry(filter) - { - let entry = result?; - - let path = entry.path(); - let metadata = entry.metadata()?; if !metadata.is_file() {