Very close to build-rs based javap

Moves javap into build.rs. Error handling works pretty well. There is one missing piece to also capture `#[java...]` derives
since those must also be reflected in build.rs.

Cargo.toml

@@ -31,6 +31,9 @@ libloading = { version = "0.8.0", optional = true }
 derive-where = "1.2.1"
 serde = { version = "1.0.214", features = ["derive"] }
 
+[build-dependencies]
+duchess-build-rs = { path = "duchess-build-rs" }
+
 
 [features]
 default = ["dylibjvm"]

build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    duchess_build_rs::DuchessBuildRs::new()
+        .with_src_path("src/".into())
+        .execute()
+        .unwrap();
+}

duchess-build-rs/Cargo.toml

@@ -9,7 +9,7 @@ documentation.workspace = true
 
 [dependencies]
 anyhow = "1.0.86"
-duchess-reflect = { version = "0.3.0", path = "../duchess-reflect" }
+duchess-reflect = { version = "0.3.0", path = "../duchess-reflect", features = ["javap-reflection"] }
 lazy_static = "1.5.0"
 proc-macro2 = "1.0.86"
 quote = "1.0.36"

duchess-build-rs/src/files.rs

@@ -7,7 +7,7 @@ pub(crate) struct File {
     pub(crate) contents: String,
 }
 
-pub fn rs_files(path: &Path) -> impl Iterator<Item = anyhow::Result<File>> {
+pub fn rs_files(path: impl AsRef<Path>) -> impl Iterator<Item = anyhow::Result<File>> {
     WalkDir::new(path)
         .into_iter()
         .filter_map(|entry| -> Option<anyhow::Result<File>> {
@@ -51,24 +51,41 @@ impl File {
     /// This is used when we are preprocessing and we find
     /// some kind of macro invocation. We want to grab all
     /// the text that may be part of it and pass it into `syn`.
+    // TODO: this should actually return an error, its basically never right to return the whole file
     pub fn rust_slice_from(&self, offset: usize) -> &str {
         let mut counter = 0;
         let terminator = self.contents[offset..].char_indices().find(|&(_, c)| {
             if c == '{' || c == '[' || c == '(' {
                 counter += 1;
             } else if c == '}' || c == ']' || c == ')' {
+                counter -= 1;
+
                 if counter == 0 {
                     return true;
                 }
-
-                counter -= 1;
             }
 
             false
         });
+        if terminator.is_none() {
+            eprintln!("rust slice ran to end of file {counter}");
+        }
         match terminator {
-            Some((i, _)) => &self.contents[offset..offset + i],
+            Some((i, _)) => &self.contents[offset..offset + i + 1],
             None => &self.contents[offset..],
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_rust_slice() {
+        for file in super::rs_files("test-files") {
+            let file = file.unwrap();
+            for offset in file.contents.char_indices().map(|(i, _)| i) {
+                let _ = file.rust_slice_from(offset);
+            }
+        }
+    }
+}

duchess-build-rs/src/impl_java_trait.rs

@@ -1,4 +1,7 @@
-use duchess_reflect::{class_info::ClassRef, reflect::Reflector};
+use duchess_reflect::{
+    class_info::ClassRef,
+    reflect::{JavapReflector, Reflect},
+};
 use proc_macro2::{Span, TokenStream};
 use syn::spanned::Spanned;
 
@@ -29,7 +32,7 @@ impl syn::parse::Parse for JavaInterfaceImpl {
 
 impl JavaInterfaceImpl {
     fn generate_shim(&self, compiler: &JavaCompiler) -> anyhow::Result<()> {
-        let reflector = Reflector::new(compiler.configuration());
+        let mut reflector = JavapReflector::new(compiler.configuration());
         let (java_interface_ref, java_interface_span) = self.java_interface()?;
         let java_interface_info =
             reflector.reflect(&java_interface_ref.name, java_interface_span)?;

duchess-build-rs/src/java_package_macro.rs

@@ -1,14 +1,54 @@
 use anyhow::Context;
-use duchess_reflect::{argument::DuchessDeclaration, parse::Parser, reflect::Reflector};
-use proc_macro2::TokenStream;
+use duchess_reflect::{argument::DuchessDeclaration, parse::Parser, reflect::JavapReflector};
+use proc_macro2::{Span, TokenStream};
 
-use crate::{files::File, java_compiler::JavaCompiler};
+use crate::{files::File, java_package_macro, re};
 
-pub fn process_macro(compiler: &JavaCompiler, file: &File, offset: usize) -> anyhow::Result<()> {
-    let the_impl: JavaPackageMacro = syn::parse_str(file.rust_slice_from(offset))
-        .with_context(|| format!("{} failed to parse java_package macro", file.slug(offset),))?;
+pub fn process_file(rs_file: &File, reflector: &mut JavapReflector) -> anyhow::Result<bool> {
+    let mut watch_file = false;
+    for capture in re::java_package().captures_iter(&rs_file.contents) {
+        eprintln!("Debug: found java macro in {:?}", rs_file.path);
+        let std::ops::Range { start, end: _ } = capture.get(0).unwrap().range();
+        java_package_macro::process_macro(reflector, &rs_file, start)
+            .with_context(|| format!("failed to process macro {}", rs_file.slug(start)))?;
+        watch_file = true;
+    }
+    Ok(watch_file)
+}
+
+fn process_macro(reflector: &mut JavapReflector, file: &File, offset: usize) -> anyhow::Result<()> {
+    let the_impl: JavaPackageMacro = match syn::parse_str(file.rust_slice_from(offset))
+        .with_context(|| {
+            format!(
+                "{} failed to parse java_package macro as Rust code",
+                file.slug(offset),
+            )
+        })
+        .with_context(|| format!("full contents:\n>>>>{}<<<", file.rust_slice_from(offset)))
+    {
+        Ok(package) => package,
+        Err(e) => {
+            // we'll let rustc deal with this later
+            eprintln!(
+                "Warning: failed to parse java_package macro as Rust code, ignoring it. Error: {}",
+                e
+            );
+            return Ok(());
+        }
+    };
 
-    the_impl.parse_contents(compiler)?;
+    let contents = match the_impl.parse_contents() {
+        Ok(decl) => decl,
+        Err(e) => {
+            // we'll let rustc deal with this later
+            eprintln!(
+                "Warning: failed to parse java_package macro as Duchess code, ignoring it. Error: {}",
+                e
+            );
+            return Ok(());
+        }
+    };
+    cache_all_classes(contents, reflector).with_context(|| "failed to execute javap")?;
     Ok(())
 }
 
@@ -30,12 +70,38 @@ impl syn::parse::Parse for JavaPackageMacro {
 }
 
 impl JavaPackageMacro {
-    fn parse_contents(self, compiler: &JavaCompiler) -> anyhow::Result<()> {
+    fn parse_contents(self) -> anyhow::Result<DuchessDeclaration> {
         let input = self.invocation.mac.tokens;
-        let decl = Parser::from(input).parse::<DuchessDeclaration>()?;
-        let mut reflector = Reflector::new(compiler.configuration());
-        let root_map = decl.to_root_map(&mut reflector)?;
-        root_map.
-        Ok(())
+        Ok(Parser::from(input).parse::<DuchessDeclaration>()?)
+    }
+}
+
+fn cache_all_classes(
+    decl: DuchessDeclaration,
+    reflector: &mut JavapReflector,
+) -> anyhow::Result<()> {
+    let _root_map = decl.to_root_map(reflector)?;
+    for class in _root_map.class_names() {
+        // forcibly reflect every class
+        use duchess_reflect::reflect::Reflect;
+        reflector.reflect(&class, Span::call_site())?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod test {
+    use crate::JavaCompiler;
+    use duchess_reflect::config::Configuration;
+    use tempfile::tempdir;
+
+    #[test]
+    fn process_file() {
+        let compiler = &JavaCompiler::new(&Configuration::new(), None).unwrap();
+        let rs_file = crate::files::File {
+            path: "test-files/java_package_1.rs".into(),
+            contents: include_str!("../test-files/java_package_1.rs").to_string(),
+        };
+        super::process_file(&rs_file, &compiler).unwrap();
     }
 }

duchess-build-rs/src/lib.rs

@@ -1,5 +1,7 @@
 use std::path::{Path, PathBuf};
 
+use anyhow::Context;
+use duchess_reflect::reflect::JavapReflector;
 use java_compiler::JavaCompiler;
 
 mod code_writer;
@@ -19,11 +21,11 @@ pub use duchess_reflect::config::Configuration;
 ///
 /// The simplest build.rs is as follows.
 ///
-/// ```rust
+/// ```rust,no_run
 /// use duchess_build_rs::DuchessBuildRs;
 ///
 /// fn main() -> anyhow::Result<()> {
-///     DuchessBuildRs::new().execute()?;
+///     DuchessBuildRs::new().execute()
 /// }
 /// ```
 pub struct DuchessBuildRs {
@@ -79,27 +81,47 @@ impl DuchessBuildRs {
     ///
     /// NB: Duchess macros must be written like `duchess::name!` or `#[duchess::name]`.
     pub fn execute(self) -> anyhow::Result<()> {
+        // TODO: Russell November 8th.
+        // Things are very close to done.
+        // We need to capture things like #[java(java.lang.Throwable)] and reflect those to store the types
+        // in the reflection cache.
+
+        // You can see this failure currently if you run `just test`.
+
         let compiler = &JavaCompiler::new(&self.configuration, self.temporary_dir.as_ref())?;
+        eprintln!(
+            "looking for files in {:?} (total: {})",
+            self.src_path,
+            files::rs_files(&self.src_path).count()
+        );
+        let mut reflector = JavapReflector::new(&self.configuration);
         for rs_file in files::rs_files(&self.src_path) {
             let rs_file = rs_file?;
             let mut watch_file = false;
 
-            for capture in re::java_package().captures_iter(&rs_file.contents) {
-                let std::ops::Range { start, end: _ } = capture.get(0).unwrap().range();
-                java_package_macro::process_macro(compiler, &rs_file, start)?;
-                watch_file = true;
-            }
+            eprintln!("looking for java macros in {:?}", rs_file.path);
+            watch_file |= java_package_macro::process_file(&rs_file, &mut reflector)?;
 
             for capture in re::impl_java_interface().captures_iter(&rs_file.contents) {
                 let std::ops::Range { start, end: _ } = capture.get(0).unwrap().range();
-                impl_java_trait::process_impl(compiler, &rs_file, start)?;
+                impl_java_trait::process_impl(compiler, &rs_file, start)
+                    .with_context(|| "failed to parse impl")?;
                 watch_file = true;
             }
 
             if watch_file && self.in_cargo {
                 println!("cargo:rerun-if-changed={}", rs_file.path.display());
             }
         }
+        let out_dir = std::env::var("OUT_DIR").unwrap();
+        eprintln!("dumping {} classes to {out_dir}", reflector.len());
+        reflector.dump_to(Path::new(&out_dir))?;
+        println!("cargo::rustc-env=DUCHESS_OUT_DIR={}", out_dir);
+        if let Some(classpath) = self.configuration.classpath() {
+            println!("cargo::rustc-env=CLASSPATH={}", classpath);
+        } else {
+            println!("cargo::rustc-env=CLASSPATH={}", out_dir);
+        }
         Ok(())
     }
 }

duchess-build-rs/src/re.rs

@@ -12,4 +12,26 @@ macro_rules! declare_regex {
 
 declare_regex!(impl_java_interface() = r"#\[duchess::impl_java_interface\]");
 
-declare_regex!(java_package() = r"duchess::java_package! *\{");
+declare_regex!(java_package() = r"(?m)^\s*(duchess|duchess_macro)::java_package! *\{");
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_java_package_regex() {
+        assert!(super::java_package().is_match(r#"       duchess_macro::java_package! { "#));
+        let java_file = r#"
+    NB. in doctests, the current crate is already available as duchess.
+
+    duchess_macro::java_package! {
+        package java.lang;
+
+        public class java.lang.Object {
+            public java.lang.Object();
+            public native int hashCode();
+            public boolean equals(java.lang.Object);
+            public java.lang.String toString();
+            public final native void notify();
+            public final native void notifyAll();"#;
+        assert!(super::java_package().is_match(java_file));
+    }
+}

duchess-reflect/Cargo.toml

@@ -8,6 +8,9 @@ homepage.workspace = true
 documentation.workspace = true
 description = "Internal component of duchess crate"
 
+[features]
+javap-reflection = []
+
 [dependencies]
 anyhow = "1.0.70"
 lalrpop-util = { version = "0.19.9", features = ["lexer"] }
@@ -16,6 +19,7 @@ proc-macro2 = "1.0.56"
 quote = "1.0.26"
 rust-format = { version = "0.3.4", features = ["token_stream"] }
 serde = { version = "1.0.214", features = ["derive", "rc"] }
+serde_json = "1.0.132"
 str_inflector = "0.12.0"
 syn = "2.0.15"
 tempfile = "3.8.1"

duchess-reflect/src/check.rs

@@ -2,11 +2,11 @@ use std::collections::HashSet;
 
 use crate::{
     class_info::{ClassInfo, ClassRef, Constructor, Flags, Method, RefType, RootMap, Type},
-    reflect::{JavapClassInfo, Reflector},
+    reflect::{JavapClassInfo, PrecomputedReflector},
 };
 
 impl RootMap {
-    pub fn check(&self, reflector: &mut Reflector) -> syn::Result<()> {
+    pub fn check(&self, reflector: &PrecomputedReflector) -> syn::Result<()> {
         let mut errors = vec![];
 
         for class_name in &self.class_names() {
@@ -28,7 +28,7 @@ impl ClassInfo {
     fn check(
         &self,
         root_map: &RootMap,
-        reflector: &mut Reflector,
+        reflector: &PrecomputedReflector,
         push_error: &mut dyn FnMut(syn::Error),
     ) -> syn::Result<()> {
         let info = reflector.reflect(&self.name, self.span)?;

duchess-reflect/src/class_info.rs

@@ -723,12 +723,31 @@ impl std::fmt::Display for Id {
 }
 
 /// A dotted identifier
-#[derive(Eq, Hash, Ord, PartialEq, PartialOrd, Clone, Debug, Deserialize, Serialize)]
+#[derive(Eq, Hash, Ord, PartialEq, PartialOrd, Clone, Debug)]
 pub struct DotId {
     /// Dotted components. Invariant: len >= 1.
     ids: Vec<Id>,
 }
 
+impl Serialize for DotId {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&self.with_sep("."))
+    }
+}
+
+impl<'de> Deserialize<'de> for DotId {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        Ok(DotId::parse(&s))
+    }
+}
+
 impl From<Id> for DotId {
     fn from(value: Id) -> Self {
         DotId { ids: vec![value] }