From 3b2dc2ffdfae47ab4bf3b2d5ac79f8f004e7065c Mon Sep 17 00:00:00 2001 From: Nico Wagner Date: Mon, 5 Feb 2024 11:10:11 +0100 Subject: [PATCH] move filter-list into `pica-toolki` --- .github/workflows/ci.yml | 14 ++++++-------- crates/pica-toolkit/Cargo.toml | 3 ++- crates/pica-toolkit/src/commands/filter.rs | 2 +- crates/pica-toolkit/src/commands/select.rs | 3 ++- .../src/filter_list.rs | 12 ++++++------ crates/pica-toolkit/src/main.rs | 1 + crates/pica-utils/Cargo.toml | 2 -- crates/pica-utils/src/lib.rs | 2 -- 8 files changed, 18 insertions(+), 21 deletions(-) rename crates/{pica-utils => pica-toolkit}/src/filter_list.rs (94%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72aa1b34e..8741f0df1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,19 +120,17 @@ jobs: cross build --target ${{ matrix.target.triple }} fuzz: - # if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }} - # see https://github.com/deutsche-nationalbibliothek/pica-rs/actions/runs/7736509249/job/21094015127 - if: false + if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }} name: fuzz ${{ matrix.item.name }} runs-on: ubuntu-latest needs: check strategy: matrix: item: - - { name: pica-record-ref, fuzz-dir: crates/pica-record/fuzz, target: fuzz-record-ref, max-total-time: 300 } - - { name: pica-record-matcher, fuzz-dir: crates/pica-matcher/fuzz, target: fuzz-record-matcher, max-total-time: 300 } - - { name: pica-select-query, fuzz-dir: crates/pica-select/fuzz, target: fuzz-query, max-total-time: 300 } - - { name: pica-path, fuzz-dir: crates/pica-path/fuzz, target: fuzz-path, max-total-time: 300 } + - { name: pica-record-ref, fuzz-dir: crates/pica-record/fuzz, target: fuzz-record-ref, max-total-time: 240 } + - { name: pica-record-matcher, fuzz-dir: crates/pica-matcher/fuzz, target: fuzz-record-matcher, max-total-time: 240 } + - { name: pica-select-query, fuzz-dir: crates/pica-select/fuzz, target: fuzz-query, max-total-time: 240 } + - { name: pica-path, fuzz-dir: crates/pica-path/fuzz, target: fuzz-path, max-total-time: 240 } steps: - uses: actions/checkout@v4 - uses: abbbi/github-actions-tune@v1 @@ -356,7 +354,7 @@ jobs: - cross - deny - fmt - # - fuzz + - fuzz - miri - test # - udeps diff --git a/crates/pica-toolkit/Cargo.toml b/crates/pica-toolkit/Cargo.toml index b2299e35d..b166ad520 100644 --- a/crates/pica-toolkit/Cargo.toml +++ b/crates/pica-toolkit/Cargo.toml @@ -16,11 +16,12 @@ csv = { workspace = true } directories = { version = "5.0" } flate2 = { workspace = true } indicatif = { version = "0.17" } -pica-utils = { workspace = true } pica-matcher = { workspace = true } pica-path = { workspace = true } pica-record = { workspace = true } pica-select = { workspace = true } +pica-utils = { workspace = true } +polars = { workspace = true } quick-xml = { version = "0.31" } rand = { workspace = true } regex = { workspace = true } diff --git a/crates/pica-toolkit/src/commands/filter.rs b/crates/pica-toolkit/src/commands/filter.rs index 68d41a296..1f0d42130 100644 --- a/crates/pica-toolkit/src/commands/filter.rs +++ b/crates/pica-toolkit/src/commands/filter.rs @@ -10,10 +10,10 @@ use pica_matcher::{ }; use pica_path::PathExt; use pica_record::io::{ReaderBuilder, RecordsIterator, WriterBuilder}; -use pica_utils::FilterList; use serde::{Deserialize, Serialize}; use crate::error::CliResult; +use crate::filter_list::FilterList; use crate::progress::Progress; use crate::{gzip_flag, skip_invalid_flag, Config}; diff --git a/crates/pica-toolkit/src/commands/select.rs b/crates/pica-toolkit/src/commands/select.rs index 39ec738bf..2b28846bc 100644 --- a/crates/pica-toolkit/src/commands/select.rs +++ b/crates/pica-toolkit/src/commands/select.rs @@ -12,11 +12,12 @@ use pica_matcher::{MatcherBuilder, MatcherOptions}; use pica_path::PathExt; use pica_record::io::{ReaderBuilder, RecordsIterator}; use pica_select::{Query, QueryExt, QueryOptions}; -use pica_utils::{FilterList, NormalizationForm}; +use pica_utils::NormalizationForm; use serde::{Deserialize, Serialize}; use crate::config::Config; use crate::error::CliResult; +use crate::filter_list::FilterList; use crate::progress::Progress; use crate::skip_invalid_flag; diff --git a/crates/pica-utils/src/filter_list.rs b/crates/pica-toolkit/src/filter_list.rs similarity index 94% rename from crates/pica-utils/src/filter_list.rs rename to crates/pica-toolkit/src/filter_list.rs index bab9d56c2..66b5ec4a1 100644 --- a/crates/pica-utils/src/filter_list.rs +++ b/crates/pica-toolkit/src/filter_list.rs @@ -8,13 +8,13 @@ use bstr::{BStr, BString}; use polars::prelude::*; #[derive(Debug, Default)] -pub struct FilterList { +pub(crate) struct FilterList { allow: BTreeSet, deny: BTreeSet, } #[derive(Debug, thiserror::Error)] -pub enum FilterListError { +pub(crate) enum FilterListError { #[error("invalid file format (path = '{0}')")] InvalidFileFormat(String), @@ -29,11 +29,11 @@ pub enum FilterListError { } impl FilterList { - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self::default() } - pub fn check(&self, idn: Option<&BStr>) -> bool { + pub(crate) fn check(&self, idn: Option<&BStr>) -> bool { if self.allow.is_empty() && self.deny.is_empty() { return true; } @@ -49,7 +49,7 @@ impl FilterList { } } - pub fn allow( + pub(crate) fn allow( mut self, filenames: Vec, ) -> Result { @@ -68,7 +68,7 @@ impl FilterList { Ok(self) } - pub fn deny( + pub(crate) fn deny( mut self, filenames: Vec, ) -> Result { diff --git a/crates/pica-toolkit/src/main.rs b/crates/pica-toolkit/src/main.rs index d570b807e..778c9828e 100644 --- a/crates/pica-toolkit/src/main.rs +++ b/crates/pica-toolkit/src/main.rs @@ -7,6 +7,7 @@ extern crate termcolor; mod commands; mod config; mod error; +mod filter_list; mod macros; mod progress; diff --git a/crates/pica-utils/Cargo.toml b/crates/pica-utils/Cargo.toml index 9a892be15..32e7b8209 100644 --- a/crates/pica-utils/Cargo.toml +++ b/crates/pica-utils/Cargo.toml @@ -10,7 +10,5 @@ rust-version.workspace = true [dependencies] bstr = { workspace = true } -polars = { workspace = true } serde = { workspace = true, features = ["derive"] } unicode-normalization = { version = "0.1.22" } -thiserror = { workspace = true } diff --git a/crates/pica-utils/src/lib.rs b/crates/pica-utils/src/lib.rs index b0d7873f7..86d4dd8f6 100644 --- a/crates/pica-utils/src/lib.rs +++ b/crates/pica-utils/src/lib.rs @@ -1,5 +1,3 @@ -mod filter_list; mod translit; -pub use filter_list::{FilterList, FilterListError}; pub use translit::NormalizationForm;