programmer/intermodal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optionally respect .gitignore in imdl torrent create

Browse Source 
Add a '--ignore' flag that, when passed, causes `imdl torretn create` to
skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and
`git config --get core.excludesFile`.

Also switches from the `walkdir` crate to the `ignore` crate, which uses
`walkdir` internally, and which handles `.gitignore` and `.ignore`
files.

This changes the behavior of `-include-hidden` on MacOS to no longer
skip entries with the hidden attribute set, due to `ignore` not exposing
`walkdir`'s filter functionality.

A PR[0] is pending to add filtering to `ignore`, so hopefully this
functionality can be re-implemented soon.

[0] https://github.com/BurntSushi/ripgrep/pull/1557

type: added
fixes:
- https://github.com/casey/intermodal/issues/378
This commit is contained in:
Celeo 2020-04-15 20:17:40 -07:00 committed by Casey Rodarmor
parent 9f48062461
commit 9b72873ed1
No known key found for this signature in database
GPG Key ID: 556186B153EC6FE0
16 changed files with 177 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -2,9 +2,10 @@ Changelog
=========
UNRELEASED - 2020-04-20
UNRELEASED - 2020-04-21
-----------------------
- :books: [`xxxxxxxxxxxx`](https://github.com/casey/intermodal/commits/master) Improve FAQ template - _Casey Rodarmor <casey@rodarmor.com>_
- :sparkles: [`xxxxxxxxxxxx`](https://github.com/casey/intermodal/commits/master) Optionally respect `.gitignore` in `imdl torrent create` - Fixes [#378](https://github.com/casey/intermodal/issues/378) - _Celeo <celeodor@gmail.com>_
- :books: [`9f480624616b`](https://github.com/casey/intermodal/commit/9f480624616b77995befec722effda22cc2d06ad) Improve FAQ template - _Casey Rodarmor <casey@rodarmor.com>_
- :wrench: [`1380290eb8e2`](https://github.com/casey/intermodal/commit/1380290eb8e222605f368bc8346a1e63c83d9af7) Make `publish-check` recipe stricter - _Casey Rodarmor <casey@rodarmor.com>_

42
Cargo.lock generated
View File

@ -240,6 +240,27 @@ dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "crossbeam-channel"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
dependencies = [
"crossbeam-utils",
"maybe-uninit",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"cfg-if",
"lazy_static",
]
[[package]]
name = "ctor"
version = "0.1.13"
@ -445,6 +466,25 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "ignore"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddf60d063dbe6b75388eec66cfc07781167ae3d34a09e0c433e6c5de0511f7fb"
dependencies = [
"crossbeam-channel",
"crossbeam-utils",
"globset",
"lazy_static",
"log",
"memchr",
"regex",
"same-file",
"thread_local",
"walkdir",
"winapi-util",
]
[[package]]
name = "imdl"
version = "0.1.6"
@ -456,6 +496,7 @@ dependencies = [
"claim",
"console",
"globset",
"ignore",
"imdl-indicatif",
"lazy_static",
"libc",
@ -480,7 +521,6 @@ dependencies = [
"temptree",
"unicode-width",
"url",
"walkdir",
]
[[package]]

2
Cargo.toml
View File

@ -21,6 +21,7 @@ globset = "0.4.0"
lazy_static = "1.4.0"
libc = "0.2.0"
log = "0.4.8"
ignore = "0.4.14"
md5 = "0.7.0"
open = "1.4.0"
pretty_assertions = "0.6.0"
@ -38,7 +39,6 @@ syn = "1.0.14"
tempfile = "3.0.0"
unicode-width = "0.1.0"
url = "2.1.1"
walkdir = "2.1.0"
[dependencies.bendy]
version = "0.3.0"

3
book/src/commands/imdl-torrent-create.md
View File

@ -14,6 +14,9 @@ FLAGS:
-f, --force Overwrite the destination `.torrent` file, if it
exists.
--help Print help message.
--ignore Skip files listed in `.gitignore`, `.ignore`,
`.git/info/exclude`, and `git config --get
core.excludesFile`.
-h, --include-hidden Include hidden files that would otherwise be
skipped, such as files that start with a `.`, and
files hidden by file attributes on macOS and

1
completions/_imdl
View File

@ -127,6 +127,7 @@ Sort in ascending order by size, break ties in descending path order:
'--private[Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.]' \
'-S[Display information about created torrent file.]' \
'--show[Display information about created torrent file.]' \
'--ignore[Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.]' \
'--help[Print help message.]' \
'-V[Print version number.]' \
'--version[Print version number.]' \

1
completions/_imdl.ps1
View File

@ -125,6 +125,7 @@ Sort in ascending order by size, break ties in descending path order:
[CompletionResult]::new('--private', 'private', [CompletionResultType]::ParameterName, 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.')
[CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Display information about created torrent file.')
[CompletionResult]::new('--show', 'show', [CompletionResultType]::ParameterName, 'Display information about created torrent file.')
[CompletionResult]::new('--ignore', 'ignore', [CompletionResultType]::ParameterName, 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.')
[CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help message.')
[CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Print version number.')
[CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Print version number.')

2
completions/imdl.bash
View File

@ -135,7 +135,7 @@ _imdl() {
return 0
;;
imdl__torrent__create)
opts=" -n -F -f -h -j -M -O -P -S -V -a -A -t -c -g -i -N -o -p -s --dry-run --follow-symlinks --force --include-hidden --include-junk --link --md5 --no-created-by --no-creation-date --open --private --show --help --version --announce --allow --announce-tier --comment --node --glob --input --name --sort-by --output --peer --piece-length --source "
opts=" -n -F -f -h -j -M -O -P -S -V -a -A -t -c -g -i -N -o -p -s --dry-run --follow-symlinks --force --include-hidden --include-junk --link --md5 --no-created-by --no-creation-date --open --private --show --ignore --help --version --announce --allow --announce-tier --comment --node --glob --input --name --sort-by --output --peer --piece-length --source "
if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0

1
completions/imdl.elvish
View File

@ -118,6 +118,7 @@ Sort in ascending order by size, break ties in descending path order:
cand --private 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.'
cand -S 'Display information about created torrent file.'
cand --show 'Display information about created torrent file.'
cand --ignore 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.'
cand --help 'Print help message.'
cand -V 'Print version number.'
cand --version 'Print version number.'

1
completions/imdl.fish
View File

@ -66,6 +66,7 @@ complete -c imdl -n "__fish_seen_subcommand_from create" -l no-creation-date -d
complete -c imdl -n "__fish_seen_subcommand_from create" -s O -l open -d 'Open `.torrent` file after creation. Uses `xdg-open`, `gnome-open`, or `kde-open` on Linux; `open` on macOS; and `cmd /C start` on Windows'
complete -c imdl -n "__fish_seen_subcommand_from create" -s P -l private -d 'Set the `private` flag. Torrent clients that understand the flag and participate in the swarm of a torrent with the flag set will only announce themselves to the announce URLs included in the torrent, and will not use other peer discovery mechanisms, such as the DHT or local peer discovery. See BEP 27: Private Torrents for more information.'
complete -c imdl -n "__fish_seen_subcommand_from create" -s S -l show -d 'Display information about created torrent file.'
complete -c imdl -n "__fish_seen_subcommand_from create" -l ignore -d 'Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config --get core.excludesFile`.'
complete -c imdl -n "__fish_seen_subcommand_from create" -l help -d 'Print help message.'
complete -c imdl -n "__fish_seen_subcommand_from create" -s V -l version -d 'Print version number.'
complete -c imdl -n "__fish_seen_subcommand_from link" -s i -l input -d 'Generate magnet link from metainfo at `PATH`. If `PATH` is `-`, read metainfo from standard input.'

4
man/imdl-torrent-create.1
View File

@ -21,6 +21,10 @@ Overwrite the destination `.torrent` file, if it exists.
\fB\-\-help\fR
Print help message.
.TP
\fB\-\-ignore\fR
Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config \fB\-\-get\fR
core.excludesFile`.
.TP
\fB\-h\fR, \fB\-\-include\-hidden\fR
Include hidden files that would otherwise be skipped, such as files that start with a `.`,
and files hidden by file attributes on macOS and Windows.

2
src/common.rs
View File

@ -27,6 +27,7 @@ pub(crate) use std::{
pub(crate) use bendy::{decoding::FromBencode, encoding::ToBencode, value::Value};
pub(crate) use chrono::{TimeZone, Utc};
pub(crate) use globset::{Glob, GlobMatcher};
pub(crate) use ignore::WalkBuilder;
pub(crate) use indicatif::{ProgressBar, ProgressStyle};
pub(crate) use libc::EXIT_FAILURE;
pub(crate) use regex::{Regex, RegexSet};
@ -44,7 +45,6 @@ pub(crate) use strum::{IntoEnumIterator, VariantNames};
pub(crate) use strum_macros::{EnumIter, EnumString, EnumVariantNames, IntoStaticStr};
pub(crate) use unicode_width::UnicodeWidthStr;
pub(crate) use url::{Host, Url};
pub(crate) use walkdir::WalkDir;
// logging functions
#[allow(unused_imports)]

17
src/error.rs
View File

@ -26,6 +26,8 @@ pub(crate) enum Error {
FileOrderUnknown { text: String },
#[snafu(display("I/O error at `{}`: {}", path.display(), source))]
Filesystem { source: io::Error, path: PathBuf },
#[snafu(display("Error searching for files: {}", source))]
FileSearch { source: ignore::Error },
#[snafu(display("Invalid glob: {}", source))]
GlobParse { source: globset::Error },
#[snafu(display("Failed to serialize torrent info dictionary: {}", source))]
@ -171,17 +173,10 @@ impl From<SystemTimeError> for Error {
}
}
impl From<walkdir::Error> for Error {
fn from(walkdir_error: walkdir::Error) -> Self {
let path = walkdir_error
.path()
.invariant_unwrap("Walkdir errors always have path")
.to_owned();
if let Some(source) = walkdir_error.into_io_error() {
Self::Filesystem { source, path }
} else {
Self::internal("Encountered walkdir error without source")
impl From<ignore::Error> for Error {
fn from(ignore_error: ignore::Error) -> Self {
Self::FileSearch {
source: ignore_error,
}
}
}

106
src/subcommand/torrent/create.rs
View File

@ -240,6 +240,12 @@ Sort in ascending order by size, break ties in descending path order:
download and upload statistics to multiple trackers."
)]
source: Option<String>,
#[structopt(
long = "ignore",
help = "Skip files listed in `.gitignore`, `.ignore`, `.git/info/exclude`, and `git config \
--get core.excludesFile`."
)]
ignore: bool,
}
impl Create {
@ -1679,12 +1685,6 @@ Content Size 9 bytes
.arg(env.resolve("foo/hidden")?)
.status()
.unwrap();
} else if cfg!(target_os = "macos") {
Command::new("chflags")
.arg("hidden")
.arg(env.resolve("foo/hidden")?)
.status()
.unwrap();
} else {
fs::remove_file(env.resolve("foo/hidden")?).unwrap();
}
@ -1905,6 +1905,7 @@ Content Size 9 bytes
);
assert_eq!(metainfo.info.pieces, PieceList::new());
}
#[test]
fn skip_hidden_attribute_dir_contents() -> Result<()> {
let mut env = test_env! {
@ -1935,17 +1936,6 @@ Content Size 9 bytes
.unwrap();
}
#[cfg(target_os = "macos")]
{
env.write("foo/bar/baz", "baz");
let path = env.resolve("foo/bar")?;
Command::new("chflags")
.arg("hidden")
.arg(&path)
.status()
.unwrap();
}
env.assert_ok();
let metainfo = env.load_metainfo("foo.torrent");
assert_matches!(
@ -2181,6 +2171,88 @@ Content Size 9 bytes
assert_eq!(metainfo.info.pieces, PieceList::from_pieces(&["a"]));
}
#[test]
fn ignore_files_in_gitignore() {
let mut env = test_env! {
args: [
"torrent",
"create",
"--input",
"foo",
"--ignore",
],
tree: {
foo: {
".gitignore": "a",
a: "a",
b: "b",
},
}
};
env.assert_ok();
let metainfo = env.load_metainfo("foo.torrent");
assert_matches!(
metainfo.info.mode,
Mode::Multiple { files } if files.len() == 1
);
}
#[test]
fn ignore_files_in_ignore() {
let mut env = test_env! {
args: [
"torrent",
"create",
"--input",
"foo",
"--ignore",
],
tree: {
foo: {
".ignore": "a",
a: "a",
b: "b",
},
}
};
env.assert_ok();
let metainfo = env.load_metainfo("foo.torrent");
assert_matches!(
metainfo.info.mode,
Mode::Multiple { files } if files.len() == 1
);
}
#[test]
fn ignore_files_in_git_exclude() {
let mut env = test_env! {
args: [
"torrent",
"create",
"--input",
"foo",
"--ignore",
],
tree: {
foo: {
".git": {
info: {
exclude: "a",
},
},
a: "a",
b: "b",
},
}
};
env.assert_ok();
let metainfo = env.load_metainfo("foo.torrent");
assert_matches!(
metainfo.info.mode,
Mode::Multiple { files } if files.len() == 1
);
}
#[test]
fn nodes_default() {
let mut env = test_env! {

1
src/subcommand/torrent/create/create_content.rs
View File

@ -27,6 +27,7 @@ impl CreateContent {
let files = Walker::new(&env.resolve(path)?)
.include_junk(create.include_junk)
.include_hidden(create.include_hidden)
.ignore(create.ignore)
.follow_symlinks(create.follow_symlinks)
.sort_by(create.sort_by.clone())
.globs(&create.globs)?

7
src/subcommand/torrent/stats.rs
View File

@ -53,7 +53,12 @@ impl Stats {
let mut extractor = Extractor::new(self.print, &self.extract_patterns);
for result in WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name())) {
for result in WalkBuilder::new(path)
.standard_filters(false)
.hidden(true)
.sort_by_file_name(|a, b| a.cmp(b))
.build()
{
if extractor.torrents >= self.limit.unwrap_or(u64::max_value()) {
break;
}

44
src/walker.rs
View File

@ -12,6 +12,7 @@ pub(crate) struct Walker {
follow_symlinks: bool,
include_hidden: bool,
include_junk: bool,
ignore: bool,
sort_by: Vec<SortSpec>,
patterns: Vec<Pattern>,
root: PathBuf,
@ -24,6 +25,7 @@ impl Walker {
follow_symlinks: false,
include_hidden: false,
include_junk: false,
ignore: false,
sort_by: Vec::new(),
patterns: Vec::new(),
root: root.to_owned(),
@ -45,6 +47,10 @@ impl Walker {
}
}
pub(crate) fn ignore(self, ignore: bool) -> Self {
Self { ignore, ..self }
}
pub(crate) fn sort_by(self, sort_by: Vec<SortSpec>) -> Self {
Self { sort_by, ..self }
}
@ -94,7 +100,17 @@ impl Walker {
return Ok(Files::file(self.root, Bytes::from(root_metadata.len())));
}
let filter = |entry: &walkdir::DirEntry| {
let mut file_infos = Vec::new();
let mut total_size = 0;
let mut walk_builder = WalkBuilder::new(&self.root);
walk_builder
.follow_links(self.follow_symlinks)
.standard_filters(self.ignore)
.require_git(false)
.hidden(!self.include_hidden);
for result in walk_builder.build() {
let entry = result?;
let path = entry.path();
if let Some(s) = &self.spinner {
@ -103,32 +119,6 @@ impl Walker {
s.tick();
}
let file_name = entry.file_name();
if !self.include_hidden && file_name.to_string_lossy().starts_with('.') {
return false;
}
let hidden = Platform::hidden(path).unwrap_or(true);
if !self.include_hidden && hidden {
return false;
}
true
};
let mut file_infos = Vec::new();
let mut total_size = 0;
for result in WalkDir::new(&self.root)
.follow_links(self.follow_symlinks)
.into_iter()
.filter_entry(filter)
{
let entry = result?;
let path = entry.path();
let metadata = entry.metadata()?;
if !metadata.is_file() {