From: Huck Boles <huck@huck.website>
Date: Sun, 28 May 2023 02:12:18 +0000 (-0500)
Subject: added: parallel iterators
X-Git-Url: https://git.huck.website/?a=commitdiff_plain;h=73e141a7880234ad494c6291399eeb9bf1a4a663;p=metaforge.git

added: parallel iterators
---

diff --git a/Cargo.lock b/Cargo.lock
index b30d11b..f9c80f5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -524,6 +524,7 @@ dependencies = [
  "pandoc",
  "pest",
  "pest_derive",
+ "rayon",
  "thiserror",
 ]
 
@@ -785,9 +786,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "2.0.17"
+version = "2.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45b6ddbb36c5b969c182aec3c4a0bce7df3fbad4b77114706a49aacc80567388"
+checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/Cargo.toml b/Cargo.toml
index 175a564..955fee0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ thiserror = "1"
 eyre = "0.6"
 pest = "2"
 pest_derive = "2"
+rayon = "1.7"
 
 [dev-dependencies]
 criterion = "0.4"
@@ -25,3 +26,7 @@ harness = false
 [[bench]]
 name = "pandoc"
 harness = false
+
+[[bench]]
+name = "parallel"
+harness = false
diff --git a/benches/parallel.rs b/benches/parallel.rs
new file mode 100644
index 0000000..b31ed61
--- /dev/null
+++ b/benches/parallel.rs
@@ -0,0 +1,32 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+pub fn parallel_build_dir(c: &mut Criterion) {
+    let dir = std::path::PathBuf::from("files/bench_site")
+        .canonicalize()
+        .unwrap();
+
+    let mut opts = metaforge::Options::new();
+    opts.root = dir.clone();
+    opts.source = dir.join("source");
+    opts.build = dir.join("build");
+    opts.pattern = dir.join("pattern");
+    opts.clean = true;
+    opts.parallel = true;
+
+    c.bench_function("build dir", |b| {
+        if opts.build.exists() {
+            std::fs::remove_dir_all(&opts.build).expect("clean build dir");
+        }
+
+        std::fs::create_dir(&opts.build).expect("create build dir");
+        b.iter(|| metaforge::build_site(black_box(&opts)).unwrap())
+    });
+}
+
+criterion_group! {
+    name = benches;
+    config = Criterion::default().sample_size(10).measurement_time(core::time::Duration::from_secs(135));
+    targets = parallel_build_dir
+}
+
+criterion_main!(benches);
diff --git a/src/lib.rs b/src/lib.rs
index 86a4eb2..d4d4da4 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,6 +5,7 @@ extern crate pest_derive;
 mod error;
 mod metafile;
 mod options;
+mod parallel;
 mod parser;
 
 #[cfg(test)]
@@ -13,6 +14,7 @@ mod tests;
 pub use error::*;
 pub use metafile::*;
 pub use options::*;
+pub use parallel::*;
 pub use parser::*;
 
 use clap::Parser;
@@ -60,7 +62,12 @@ pub fn build_site(opts: &Options) -> Result<()> {
     };
 
     source.map(&global_init)?;
-    source.build_dir()
+
+    if opts.parallel {
+        source.par_dir()
+    } else {
+        source.build_dir()
+    }
 }
 
 pub fn single_file(opts: &Options) -> Result<String> {
diff --git a/src/parallel.rs b/src/parallel.rs
new file mode 100644
index 0000000..6784015
--- /dev/null
+++ b/src/parallel.rs
@@ -0,0 +1,43 @@
+use crate::{DirNode, MetaError};
+use eyre::Result;
+use rayon::prelude::*;
+use std::fs;
+
+impl<'a> DirNode<'a> {
+    pub fn par_file(&mut self) -> Result<()> {
+        self.files.par_iter_mut().for_each(|file| {
+            file.merge(&self.global);
+            match file.construct() {
+                Ok(str) => {
+                    fs::write(file.dest().unwrap(), str).unwrap();
+                }
+                Err(e) => {
+                    // print a line to stderr about failure but continue with other files
+                    if self.opts.force {
+                        eprintln!("ignoring {}: {}", file.path.display(), e);
+                    } else {
+                        match *e {
+                            MetaError::Ignored => {}
+                            e => {
+                                eprintln!("{}", file.path.display());
+                                panic!("{}", e);
+                            }
+                        }
+                    }
+                }
+            }
+        });
+        Ok(())
+    }
+
+    pub fn par_dir(&'a mut self) -> Result<()> {
+        self.build_files()?;
+
+        self.dirs.par_iter_mut().for_each(|dir| {
+            dir.map(&self.global).unwrap();
+            dir.build_dir().unwrap();
+        });
+
+        Ok(())
+    }
+}
diff --git a/tests/build_dir.rs b/tests/build_dir.rs
index c2253c4..4874e89 100644
--- a/tests/build_dir.rs
+++ b/tests/build_dir.rs
@@ -32,3 +32,37 @@ fn build_test_site() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn parallel_build_test_site() -> Result<()> {
+    let dir = std::path::PathBuf::from("files/test_site")
+        .canonicalize()
+        .unwrap();
+
+    let mut opts = metaforge::Options::new();
+    opts.root = dir.clone();
+    opts.source = dir.join("source");
+    opts.build = dir.join("build");
+    opts.pattern = dir.join("pattern");
+    opts.clean = true;
+    opts.parallel = true;
+
+    metaforge::build_site(&opts)?;
+
+    assert!(opts.build.join("unit_tests").exists());
+    assert!(opts
+        .build
+        .join("unit_tests/blank/blank_array.html")
+        .exists());
+    assert!(opts
+        .build
+        .join("unit_tests/expand/variable_in_source.html")
+        .exists());
+    assert!(opts
+        .build
+        .join("unit_tests/override/variable.html")
+        .exists());
+    assert!(opts.build.join("unit_tests/global/pattern.html").exists());
+
+    Ok(())
+}