From cc920663dd01a0188417d1fcf3e5463c3bc8bc76 Mon Sep 17 00:00:00 2001 From: Renaud Paquay Date: Fri, 10 May 2019 11:20:21 -0700 Subject: [PATCH 1/4] Windows: Use NtQueryDirectoryFile to enumerate file entries * NtQueryDirectoryFile is more efficient than FindFindFile/FindNextFile * Include a minimal copy of "ntifs.h" containing the definitions required to call NtQueryDirectoryFile and RtlNtStatusToDosError. * Use a new set of JNI entry point to maximize performance. Instead of c++ code calling back into Java, we use a DirectByteBuffer to share native memory between C++ and Java, making the JNI interface less chatty and more efficient. This is about 20% faster than using Java callbacks. * This is available on Windows Vista/Windows Server 2008 and later. * Add entry point to check if new API is supported --- build.gradle | 3 + src/main/cpp/win.cpp | 154 ++++++++++++- .../internal/DefaultWindowsFiles.java | 206 +++++++++++++++++- .../platform/internal/WindowsDirList.java | 6 + .../internal/jni/WindowsFileFunctions.java | 8 + src/shared/headers/ntifs_min.h | 149 +++++++++++++ 6 files changed, 513 insertions(+), 13 deletions(-) create mode 100644 src/shared/headers/ntifs_min.h diff --git a/build.gradle b/build.gradle index 28d8464e..7b99f285 100755 --- a/build.gradle +++ b/build.gradle @@ -274,6 +274,9 @@ model { } else if (targetPlatform.operatingSystem.windows) { if (name.contains("_min")) { cppCompiler.define "WINDOWS_MIN" + } else { + // For NtQueryDirectoryFile + linker.args "ntdll.lib" } cppCompiler.args "-I${org.gradle.internal.jvm.Jvm.current().javaHome}/include" cppCompiler.args "-I${org.gradle.internal.jvm.Jvm.current().javaHome}/include/win32" diff --git a/src/main/cpp/win.cpp b/src/main/cpp/win.cpp index df0a1ac5..94489b36 100755 --- a/src/main/cpp/win.cpp +++ b/src/main/cpp/win.cpp @@ -21,6 +21,9 @@ #include #include #include +#ifndef WINDOWS_MIN +#include "ntifs_min.h" +#endif #define ALL_COLORS (FOREGROUND_BLUE|FOREGROUND_RED|FOREGROUND_GREEN) @@ -31,8 +34,18 @@ void mark_failed_with_errno(JNIEnv *env, const char* message, jobject result) { mark_failed_with_code(env, message, GetLastError(), NULL, result); } +#ifndef WINDOWS_MIN +/* + * Marks the given result as failed, using a NTSTATUS error code + */ +void mark_failed_with_ntstatus(JNIEnv *env, const char* message, NTSTATUS status, jobject result) { + ULONG win32ErrorCode = RtlNtStatusToDosError(status); + mark_failed_with_code(env, message, win32ErrorCode, NULL, result); +} +#endif + int map_error_code(int error_code) { - if (error_code == ERROR_PATH_NOT_FOUND) { + if (error_code == ERROR_FILE_NOT_FOUND || error_code == ERROR_PATH_NOT_FOUND) { return FAILURE_NO_SUCH_FILE; } if (error_code == ERROR_DIRECTORY) { @@ -81,6 +94,7 @@ bool is_path_absolute_unc(wchar_t* path, int path_len) { // // Returns a UTF-16 string that is the concatenation of |prefix| and |path|. +// The string must be deallocated with a call to free(). // wchar_t* add_prefix(wchar_t* path, int path_len, wchar_t* prefix) { int prefix_len = wcslen(prefix); @@ -156,10 +170,6 @@ jlong lastModifiedNanos(FILETIME* time) { return ((jlong)time->dwHighDateTime << 32) | time->dwLowDateTime; } -jlong lastModifiedNanos(LARGE_INTEGER* time) { - return ((jlong)time->HighPart << 32) | time->LowPart; -} - // // Retrieves the file attributes for the file specified by |pathStr|. // If |followLink| is true, symbolic link targets are resolved. @@ -593,6 +603,140 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_readdir(JNIEn FindClose(dirHandle); } +// +// Returns "true" is the various fastReaddirXxx calls are supported on this platform. +// +JNIEXPORT jboolean JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirIsSupported(JNIEnv *env, jclass target) { +#ifdef WINDOWS_MIN + return JNI_FALSE; +#else + return JNI_TRUE; +#endif +} + +#ifndef WINDOWS_MIN +typedef struct fast_readdir_handle { + HANDLE handle; + wchar_t* pathStr; +} readdir_fast_handle_t; +#endif + +#ifndef WINDOWS_MIN +NTSTATUS invokeNtQueryDirectoryFile(HANDLE handle, BYTE* buffer, ULONG bufferSize) { + IO_STATUS_BLOCK ioStatusBlock; + + return NtQueryDirectoryFile( + handle, // FileHandle + NULL, // Event + NULL, // ApcRoutine + NULL, // ApcContext + &ioStatusBlock, // IoStatusBlock + buffer, // FileInformation + bufferSize, // Length + FileIdFullDirectoryInformation, // FileInformationClass + FALSE, // ReturnSingleEntry + NULL, // FileName + FALSE); // RestartScan +} +#endif + +// +// Returns a DirectByteBuffer pointing to a |fast_readdir_handle| structure on success +// Returns NULL on failure (and sets error message in |result|). +// +JNIEXPORT jlong JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirOpen(JNIEnv *env, jclass target, jstring path, jobject result) { +#ifdef WINDOWS_MIN + mark_failed_with_code(env, "Operation not supported", ERROR_CALL_NOT_IMPLEMENTED, NULL, result); + return NULL; +#else + // Open file for directory listing + wchar_t* pathStr = java_to_wchar_path(env, path, result); + if (pathStr == NULL) { + mark_failed_with_code(env, "Out of native memory", ERROR_OUTOFMEMORY, NULL, result); + return NULL; + } + HANDLE handle = CreateFileW(pathStr, FILE_LIST_DIRECTORY, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (handle == INVALID_HANDLE_VALUE) { + mark_failed_with_errno(env, "could not open directory", result); + free(pathStr); + return NULL; + } + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)LocalAlloc(LPTR, sizeof(readdir_fast_handle_t)); + if (readdirHandle == NULL) { + mark_failed_with_code(env, "Out of native memory", ERROR_OUTOFMEMORY, NULL, result); + CloseHandle(handle); + free(pathStr); + return NULL; + } + readdirHandle->handle = handle; + readdirHandle->pathStr = pathStr; + return (jlong)readdirHandle; +#endif +} + +// +// Releases all native resources associted to the passed in |handle| (a pointer to |fast_readdir_handle_t|). +// +JNIEXPORT void JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirClose(JNIEnv *env, jclass target, jlong handle) { +#ifdef WINDOWS_MIN + // Not supported +#else + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)handle; + CloseHandle(readdirHandle->handle); + free(readdirHandle->pathStr); + LocalFree(readdirHandle); +#endif +} + +// +// Reads the next batch of entries from the directory. +// Returns JNI_TRUE on success and if there are more entries found +// Returns JNI_FALSE and sets an error to |result| if there is an error +// Returns JNI_FALSE if there are no more entries +// +JNIEXPORT jboolean JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirNext(JNIEnv *env, jclass target, jlong handle, jobject buffer, jobject result) { +#ifdef WINDOWS_MIN + mark_failed_with_code(env, "Operation not supported", ERROR_CALL_NOT_IMPLEMENTED, NULL, result); + return JNI_FALSE; +#else + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)handle; + + BYTE* entryBuffer = (BYTE*)env->GetDirectBufferAddress(buffer); + ULONG entryBufferSize = (ULONG)env->GetDirectBufferCapacity(buffer); + + NTSTATUS status = invokeNtQueryDirectoryFile(readdirHandle->handle, entryBuffer, entryBufferSize); + if (!NT_SUCCESS(status)) { + // Normal completion: no more files in directory + if (status == STATUS_NO_MORE_FILES) { + return JNI_FALSE; + } + + /* + * NtQueryDirectoryFile returns STATUS_INVALID_PARAMETER when + * asked to enumerate an invalid directory (ie it is a file + * instead of a directory). Verify that is the actual cause + * of the error. + */ + if (status == STATUS_INVALID_PARAMETER) { + DWORD attributes = GetFileAttributesW(readdirHandle->pathStr); + if ((attributes & FILE_ATTRIBUTE_DIRECTORY) == 0) { + status = STATUS_NOT_A_DIRECTORY; + } + } + mark_failed_with_ntstatus(env, "Error reading directory entries", status, result); + return JNI_FALSE; + } + + return JNI_TRUE; +#endif +} + /* * Console functions */ diff --git a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java index 5db7579b..6eaa3958 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java @@ -24,9 +24,19 @@ import net.rubygrapefruit.platform.internal.jni.WindowsFileFunctions; import java.io.File; +import java.lang.ref.WeakReference; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.CharBuffer; import java.util.List; public class DefaultWindowsFiles extends AbstractFiles implements WindowsFiles { + private static final int FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400; + private static final int FILE_ATTRIBUTE_DIRECTORY = 0x00000010; + private static final int IO_REPARSE_TAG_SYMLINK = 0xA000000C; + + private DirectoryLister directoryLister; + public WindowsFileInfo stat(File file) throws NativeException { return stat(file, false); } @@ -41,17 +51,197 @@ public WindowsFileInfo stat(File file, boolean linkTarget) throws NativeExceptio return stat; } + public List listDir(File dir) throws NativeException { + return listDir(dir, false); + } + public List listDir(File dir, boolean linkTarget) throws NativeException { - FunctionResult result = new FunctionResult(); - WindowsDirList dirList = new WindowsDirList(); - WindowsFileFunctions.readdir(dir.getPath(), linkTarget, dirList, result); - if (result.isFailed()) { - throw listDirFailure(dir, result); + if (directoryLister == null) { + directoryLister = WindowsFileFunctions.fastReaddirIsSupported() ? new FastLister() : new BackwardCompatibleLister(); } - return dirList.files; + return directoryLister.listDir(dir, linkTarget); } - public List listDir(File dir) throws NativeException { - return listDir(dir, false); + private interface DirectoryLister { + List listDir(File dir, boolean linkTarget) throws NativeException; + } + + private class BackwardCompatibleLister implements DirectoryLister { + public List listDir(File dir, boolean linkTarget) throws NativeException { + FunctionResult result = new FunctionResult(); + WindowsDirList dirList = new WindowsDirList(); + WindowsFileFunctions.readdir(dir.getPath(), linkTarget, dirList, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + return dirList.files; + } + } + + private class FastLister implements DirectoryLister { + static final int SIZEOF_WCHAR = 2; + static final int OFFSETOF_NEXT_ENTRY_OFFSET = 0; + static final int OFFSETOF_LAST_WRITE_TIME = 24; + static final int OFFSETOF_END_OF_FILE = 40; + static final int OFFSETOF_FILE_ATTRIBUTES = 56; + static final int OFFSETOF_FILENAME_LENGTH = 60; + static final int OFFSETOF_EA_SIZE = 64; + static final int OFFSETOF_FILENAME = 80; + + /** + * We use a thread local context to store a weak reference to a {@link NtQueryDirectoryFileContext} + * instance so that we can avoid an extra direct {@link ByteBuffer} allocation at every + * invocation of {@link #listDir(File, boolean)}. + *

This has shown to improve performance by a few percents in stress test benchmarks, while at the + * same time limiting memory usage increase to a minimum (about 8KB per thread using this API)

+ *

This is safe because {@link #listDir(File, boolean)} invocations are self-contained, i.e. don't + * call back into external code.

+ */ + private final ThreadLocal> threadLocalContext = + new ThreadLocal>(); + + public List listDir(File dir, boolean linkTarget) throws NativeException { + FunctionResult result = new FunctionResult(); + WindowsDirList dirList = new WindowsDirList(); + String path = dir.getPath(); + + long handle = WindowsFileFunctions.fastReaddirOpen(path, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + try { + NtQueryDirectoryFileContext context = getNtQueryDirectoryFileContext(); + + boolean more = WindowsFileFunctions.fastReaddirNext(handle, context.buffer, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + if (more) { + int entryOffset = 0; + while (true) { + // Read entry from buffer + entryOffset = addFullDirEntry(context, dir, linkTarget, entryOffset, dirList); + + // If we reached end of buffer, fetch next set of entries + if (entryOffset == 0) { + more = WindowsFileFunctions.fastReaddirNext(handle, context.buffer, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + if (!more) { + break; + } + } + } + } + } finally { + WindowsFileFunctions.fastReaddirClose(handle); + } + + return dirList.files; + } + + /** + * Parse the content of {@link NtQueryDirectoryFileContext#buffer} at offset {@code entryOffset} + * as a FILE_ID_FULL_DIR_INFORMATION + * native structure and adds the parsed entry into the {@link WindowsDirList} collection. + *

Returns the byte offset of the next entry in {@link NtQueryDirectoryFileContext#buffer} if there is one, + * or {@code 0} if there is no next entry.

+ */ + private int addFullDirEntry(NtQueryDirectoryFileContext context, File dir, boolean followLink, int entryOffset, WindowsDirList dirList) { + // typedef struct _FILE_ID_FULL_DIR_INFORMATION { + // ULONG NextEntryOffset; // offset = 0 + // ULONG FileIndex; // offset = 4 + // LARGE_INTEGER CreationTime; // offset = 8 + // LARGE_INTEGER LastAccessTime; // offset = 16 + // LARGE_INTEGER LastWriteTime; // offset = 24 + // LARGE_INTEGER ChangeTime; // offset = 32 + // LARGE_INTEGER EndOfFile; // offset = 40 + // LARGE_INTEGER AllocationSize; // offset = 48 + // ULONG FileAttributes; // offset = 56 + // ULONG FileNameLength; // offset = 60 + // ULONG EaSize; // offset = 64 + // LARGE_INTEGER FileId; // offset = 72 + // WCHAR FileName[1]; // offset = 80 + //} FILE_ID_FULL_DIR_INFORMATION, *PFILE_ID_FULL_DIR_INFORMATION; + int nextEntryOffset = context.buffer.getInt(entryOffset + OFFSETOF_NEXT_ENTRY_OFFSET); + + long fileSize = context.buffer.getLong(entryOffset + OFFSETOF_END_OF_FILE); + long lastModified = context.buffer.getLong(entryOffset + OFFSETOF_LAST_WRITE_TIME); + + // + // See https://docs.microsoft.com/en-us/windows/desktop/fileio/reparse-point-tags + // IO_REPARSE_TAG_SYMLINK (0xA000000C) + // + int fileAttributes = context.buffer.getInt(entryOffset + OFFSETOF_FILE_ATTRIBUTES); + int reparseTagData = context.buffer.getInt(entryOffset + OFFSETOF_EA_SIZE); + + FileInfo.Type type = getFileType(fileAttributes, reparseTagData); + + int FileNameByteCount = context.buffer.getInt(entryOffset + OFFSETOF_FILENAME_LENGTH); + context.charBuffer.position((entryOffset + OFFSETOF_FILENAME) / SIZEOF_WCHAR); + context.charBuffer.get(context.fileNameArray, 0, FileNameByteCount / SIZEOF_WCHAR); + String fileName = new String(context.fileNameArray, 0, FileNameByteCount / SIZEOF_WCHAR); + + // Skip "." and ".." entries + if (!".".equals(fileName) && !"..".equals(fileName)) { + if (type == FileInfo.Type.Symlink && followLink) { + WindowsFileInfo targetInfo = stat(new File(dir, fileName), true); + dirList.addFile(fileName, targetInfo); + } else { + dirList.addFile(fileName, type.ordinal(), fileSize, lastModified); + } + } + + return nextEntryOffset == 0 ? 0 : entryOffset + nextEntryOffset; + } + + private FileInfo.Type getFileType(int dwFileAttributes, int reparseTagData) { + if (((dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT) && (reparseTagData == IO_REPARSE_TAG_SYMLINK)) { + return FileInfo.Type.Symlink; + } else if ((dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY) { + return FileInfo.Type.Directory; + } else { + return FileInfo.Type.File; + } + } + + private NtQueryDirectoryFileContext getNtQueryDirectoryFileContext() { + WeakReference ref = threadLocalContext.get(); + NtQueryDirectoryFileContext result = (ref == null ? null : ref.get()); + if (result == null) { + result = new NtQueryDirectoryFileContext(); + ref = new WeakReference(result); + threadLocalContext.set(ref); + } + return result; + } + + private class NtQueryDirectoryFileContext { + /** + * The {@code direct} {@link ByteBuffer} used to share memory between C++ and Java code. + */ + final ByteBuffer buffer; + /** + * A {@link CharBuffer} view of the {@link #buffer} above used to retrieve UTF-16 filename strings. + */ + final CharBuffer charBuffer; + /** + * A {@link java.lang.Character} array used to copy characters from the {@link #charBuffer} above + * before converting into a {@link java.lang.String} instance. + */ + final char[] fileNameArray; + + NtQueryDirectoryFileContext() { + // Note: 8KB is enough to store ~100 FILE_ID_FULL_DIR_INFORMATION entries, counting 80 bytes + // as "fixed" data plus 10 UTF-16 characters for the file name. + buffer = ByteBuffer.allocateDirect(8192); + // Win32/x86 is little endian + buffer.order(ByteOrder.LITTLE_ENDIAN); + charBuffer = buffer.asCharBuffer(); + // Note: Win32 file names are limited to 256 characters + fileNameArray = new char[256]; + } + } } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java index 319fac34..0054c925 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java @@ -16,6 +16,8 @@ package net.rubygrapefruit.platform.internal; +import net.rubygrapefruit.platform.file.WindowsFileInfo; + public class WindowsDirList extends DirList { // Called from native code @SuppressWarnings("UnusedDeclaration") @@ -23,4 +25,8 @@ public class WindowsDirList extends DirList { public void addFile(String name, int type, long size, long lastModified) { super.addFile(name, type, size, WindowsFileTime.toJavaTime(lastModified)); } + + public void addFile(String name, WindowsFileInfo fileInfo) { + super.addFile(name, fileInfo.getType().ordinal(), fileInfo.getSize(), fileInfo.getLastModifiedTime()); + } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java b/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java index a192338f..1e75e99d 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java @@ -20,8 +20,16 @@ import net.rubygrapefruit.platform.internal.FunctionResult; import net.rubygrapefruit.platform.internal.WindowsFileStat; +import java.nio.ByteBuffer; + +@SuppressWarnings("SpellCheckingInspection") public class WindowsFileFunctions { public static native void stat(String file, boolean followLink, WindowsFileStat stat, FunctionResult result); public static native void readdir(String path, boolean followLink, DirList dirList, FunctionResult result); + + public static native boolean fastReaddirIsSupported(); + public static native long fastReaddirOpen(String path, FunctionResult result); + public static native void fastReaddirClose(long handle); + public static native boolean fastReaddirNext(long handle, ByteBuffer buffer, FunctionResult result); } diff --git a/src/shared/headers/ntifs_min.h b/src/shared/headers/ntifs_min.h new file mode 100644 index 00000000..9206c38e --- /dev/null +++ b/src/shared/headers/ntifs_min.h @@ -0,0 +1,149 @@ +#ifndef _NTIFS_MIN_ +#define _NTIFS_MIN_ + +extern "C" { + +/* +* Copy necessary structures and definitions out of the Windows DDK +* to enable calling NtQueryDirectoryFile() +*/ + +typedef _Return_type_success_(return >= 0) LONG NTSTATUS; +#define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0) + +typedef struct _UNICODE_STRING { + USHORT Length; + USHORT MaximumLength; +#ifdef MIDL_PASS + [size_is(MaximumLength / 2), length_is((Length) / 2)] USHORT * Buffer; +#else // MIDL_PASS + _Field_size_bytes_part_(MaximumLength, Length) PWCH Buffer; +#endif // MIDL_PASS +} UNICODE_STRING; +typedef UNICODE_STRING *PUNICODE_STRING; +typedef const UNICODE_STRING *PCUNICODE_STRING; + +typedef enum _FILE_INFORMATION_CLASS { + FileDirectoryInformation = 1, + FileFullDirectoryInformation, + FileBothDirectoryInformation, + FileBasicInformation, + FileStandardInformation, + FileInternalInformation, + FileEaInformation, + FileAccessInformation, + FileNameInformation, + FileRenameInformation, + FileLinkInformation, + FileNamesInformation, + FileDispositionInformation, + FilePositionInformation, + FileFullEaInformation, + FileModeInformation, + FileAlignmentInformation, + FileAllInformation, + FileAllocationInformation, + FileEndOfFileInformation, + FileAlternateNameInformation, + FileStreamInformation, + FilePipeInformation, + FilePipeLocalInformation, + FilePipeRemoteInformation, + FileMailslotQueryInformation, + FileMailslotSetInformation, + FileCompressionInformation, + FileObjectIdInformation, + FileCompletionInformation, + FileMoveClusterInformation, + FileQuotaInformation, + FileReparsePointInformation, + FileNetworkOpenInformation, + FileAttributeTagInformation, + FileTrackingInformation, + FileIdBothDirectoryInformation, + FileIdFullDirectoryInformation, + FileValidDataLengthInformation, + FileShortNameInformation, + FileIoCompletionNotificationInformation, + FileIoStatusBlockRangeInformation, + FileIoPriorityHintInformation, + FileSfioReserveInformation, + FileSfioVolumeInformation, + FileHardLinkInformation, + FileProcessIdsUsingFileInformation, + FileNormalizedNameInformation, + FileNetworkPhysicalNameInformation, + FileIdGlobalTxDirectoryInformation, + FileIsRemoteDeviceInformation, + FileAttributeCacheInformation, + FileNumaNodeInformation, + FileStandardLinkInformation, + FileRemoteProtocolInformation, + FileMaximumInformation +} FILE_INFORMATION_CLASS, *PFILE_INFORMATION_CLASS; + +typedef struct _FILE_ID_FULL_DIR_INFORMATION { + ULONG NextEntryOffset; + ULONG FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + ULONG FileAttributes; + ULONG FileNameLength; + ULONG EaSize; + LARGE_INTEGER FileId; + WCHAR FileName[1]; +} FILE_ID_FULL_DIR_INFORMATION, *PFILE_ID_FULL_DIR_INFORMATION; + +typedef struct _IO_STATUS_BLOCK { + union { + NTSTATUS Status; + PVOID Pointer; + } u; + ULONG_PTR Information; +} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK; + +typedef VOID +(NTAPI *PIO_APC_ROUTINE)( + IN PVOID ApcContext, + IN PIO_STATUS_BLOCK IoStatusBlock, + IN ULONG Reserved); + +NTSYSCALLAPI +NTSTATUS +NTAPI +NtQueryDirectoryFile( + _In_ HANDLE FileHandle, + _In_opt_ HANDLE Event, + _In_opt_ PIO_APC_ROUTINE ApcRoutine, + _In_opt_ PVOID ApcContext, + _Out_ PIO_STATUS_BLOCK IoStatusBlock, + _Out_writes_bytes_(Length) PVOID FileInformation, + _In_ ULONG Length, + _In_ FILE_INFORMATION_CLASS FileInformationClass, + _In_ BOOLEAN ReturnSingleEntry, + _In_opt_ PUNICODE_STRING FileName, + _In_ BOOLEAN RestartScan +); + +// +// This function might be needed for some of the internal Windows functions, +// defined in this header file. +// +_When_(Status < 0, _Out_range_(>, 0)) +_When_(Status >= 0, _Out_range_(==, 0)) +ULONG +NTAPI +RtlNtStatusToDosError ( + NTSTATUS Status +); + +#define STATUS_NO_MORE_FILES ((NTSTATUS)0x80000006L) +#define STATUS_NOT_A_DIRECTORY ((NTSTATUS)0xC0000103L) + +} // extern "C" + +#endif // _NTIFS_MIN_ \ No newline at end of file From 32305373c4d7006dbfe445793cd7a2418849c06c Mon Sep 17 00:00:00 2001 From: Renaud Paquay Date: Wed, 7 Aug 2019 12:20:46 -0700 Subject: [PATCH 2/4] Fix typos in comments, minor code cleanup. --- src/main/cpp/win.cpp | 7 +++--- .../internal/DefaultWindowsFiles.java | 24 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/main/cpp/win.cpp b/src/main/cpp/win.cpp index 94489b36..5a103bf8 100755 --- a/src/main/cpp/win.cpp +++ b/src/main/cpp/win.cpp @@ -604,7 +604,7 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_readdir(JNIEn } // -// Returns "true" is the various fastReaddirXxx calls are supported on this platform. +// Returns "true" if the various fastReaddirXxx calls are supported on this platform. // JNIEXPORT jboolean JNICALL Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirIsSupported(JNIEnv *env, jclass target) { @@ -642,7 +642,8 @@ NTSTATUS invokeNtQueryDirectoryFile(HANDLE handle, BYTE* buffer, ULONG bufferSiz #endif // -// Returns a DirectByteBuffer pointing to a |fast_readdir_handle| structure on success +// Opens a directory for file enumeration and returns a handle to a |fast_readdir_handle| structure +// on success. The handle must be released by calling "xxx_fastReaddirClose" when done. // Returns NULL on failure (and sets error message in |result|). // JNIEXPORT jlong JNICALL @@ -679,7 +680,7 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirOp } // -// Releases all native resources associted to the passed in |handle| (a pointer to |fast_readdir_handle_t|). +// Releases all native resources associated with |handle|, a pointer to |fast_readdir_handle|. // JNIEXPORT void JNICALL Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirClose(JNIEnv *env, jclass target, jlong handle) { diff --git a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java index 6eaa3958..0f4e9016 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java @@ -31,10 +31,6 @@ import java.util.List; public class DefaultWindowsFiles extends AbstractFiles implements WindowsFiles { - private static final int FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400; - private static final int FILE_ATTRIBUTE_DIRECTORY = 0x00000010; - private static final int IO_REPARSE_TAG_SYMLINK = 0xA000000C; - private DirectoryLister directoryLister; public WindowsFileInfo stat(File file) throws NativeException { @@ -79,14 +75,18 @@ public List listDir(File dir, boolean linkTarget) throws Nat } private class FastLister implements DirectoryLister { - static final int SIZEOF_WCHAR = 2; - static final int OFFSETOF_NEXT_ENTRY_OFFSET = 0; - static final int OFFSETOF_LAST_WRITE_TIME = 24; - static final int OFFSETOF_END_OF_FILE = 40; - static final int OFFSETOF_FILE_ATTRIBUTES = 56; - static final int OFFSETOF_FILENAME_LENGTH = 60; - static final int OFFSETOF_EA_SIZE = 64; - static final int OFFSETOF_FILENAME = 80; + private static final int FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400; + private static final int FILE_ATTRIBUTE_DIRECTORY = 0x00000010; + private static final int IO_REPARSE_TAG_SYMLINK = 0xA000000C; + + private static final int SIZEOF_WCHAR = 2; + private static final int OFFSETOF_NEXT_ENTRY_OFFSET = 0; + private static final int OFFSETOF_LAST_WRITE_TIME = 24; + private static final int OFFSETOF_END_OF_FILE = 40; + private static final int OFFSETOF_FILE_ATTRIBUTES = 56; + private static final int OFFSETOF_FILENAME_LENGTH = 60; + private static final int OFFSETOF_EA_SIZE = 64; + private static final int OFFSETOF_FILENAME = 80; /** * We use a thread local context to store a weak reference to a {@link NtQueryDirectoryFileContext} From 3720c9a6e7e5f4349de09d5cbf21e0c65a5e904c Mon Sep 17 00:00:00 2001 From: Renaud Paquay Date: Mon, 20 May 2019 15:00:22 -0700 Subject: [PATCH 3/4] Windows: Add support for returning file IDs from `stat` and `readdir` * Add `FileInfo.getKey()` that returns a file identifier than can be used to compare efficiently files for equality. If the key is not available, the method returns `null`. * This is a similar concept as exposed in the Java 7 nio API https://docs.oracle.com/javase/7/docs/api/java/nio/file/attribute/BasicFileAttributes.html#fileKey() * A WindowsFileKey is a pair of (volume serial number, file id on the volume) * Update the `stat` and `fastReaddirXxx` jni entry points to provide this information * Update the test app to display `file key` if it is available --- src/main/cpp/posix.cpp | 4 ++ src/main/cpp/win.cpp | 51 +++++++++++++++++++ .../platform/file/FileInfo.java | 8 +++ .../platform/file/WindowsFileInfo.java | 9 ++++ .../internal/DefaultWindowsFiles.java | 12 +++-- .../platform/internal/DirList.java | 18 +++++-- .../platform/internal/FileStat.java | 4 ++ .../platform/internal/WindowsDirList.java | 43 ++++++++++++++-- .../platform/internal/WindowsFileKey.java | 39 ++++++++++++++ .../platform/internal/WindowsFileStat.java | 34 +++++++++++++ .../internal/jni/WindowsFileFunctions.java | 1 + src/shared/headers/generic.h | 2 + .../rubygrapefruit/platform/test/Main.java | 6 +++ 13 files changed, 222 insertions(+), 9 deletions(-) create mode 100644 src/main/java/net/rubygrapefruit/platform/internal/WindowsFileKey.java diff --git a/src/main/cpp/posix.cpp b/src/main/cpp/posix.cpp index 8682b809..56db0d0a 100755 --- a/src/main/cpp/posix.cpp +++ b/src/main/cpp/posix.cpp @@ -105,6 +105,8 @@ void unpackStat(struct stat* source, file_stat_t* result) { #else result->lastModified = toMillis(source->st_mtimespec); #endif + result->volumeId = 0; + result->fileId = 0; } JNIEXPORT void JNICALL @@ -207,6 +209,8 @@ Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_readdir(JNIEnv fileResult.fileType = FILE_TYPE_MISSING; fileResult.size = 0; fileResult.lastModified = 0; + fileResult.volumeId = 0; + fileResult.fileId = 0; } else { unpackStat(&fileInfo, &fileResult); } diff --git a/src/main/cpp/win.cpp b/src/main/cpp/win.cpp index 5a103bf8..1e8d0601 100755 --- a/src/main/cpp/win.cpp +++ b/src/main/cpp/win.cpp @@ -189,11 +189,15 @@ DWORD get_file_stat(wchar_t* pathStr, jboolean followLink, file_stat_t* pFileSta pFileStat->lastModified = 0; pFileStat->size = 0; pFileStat->fileType = FILE_TYPE_MISSING; + pFileStat->volumeId = 0; + pFileStat->fileId = 0; return ERROR_SUCCESS; } return error; } pFileStat->lastModified = lastModifiedNanos(&attr.ftLastWriteTime); + pFileStat->volumeId = 0; + pFileStat->fileId = 0; if (attr.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { pFileStat->size = 0; pFileStat->fileType = FILE_TYPE_DIRECTORY; @@ -223,6 +227,8 @@ DWORD get_file_stat(wchar_t* pathStr, jboolean followLink, file_stat_t* pFileSta pFileStat->lastModified = 0; pFileStat->size = 0; pFileStat->fileType = FILE_TYPE_MISSING; + pFileStat->volumeId = 0; + pFileStat->fileId = 0; return ERROR_SUCCESS; } return error; @@ -250,6 +256,8 @@ DWORD get_file_stat(wchar_t* pathStr, jboolean followLink, file_stat_t* pFileSta pFileStat->lastModified = lastModifiedNanos(&fileInfo.ftLastWriteTime); pFileStat->size = 0; + pFileStat->volumeId = fileInfo.dwVolumeSerialNumber; + pFileStat->fileId = ((LONGLONG)fileInfo.nFileIndexHigh << 32) | fileInfo.nFileIndexLow; if (is_file_symlink(fileTagInfo.FileAttributes, fileTagInfo.ReparseTag)) { pFileStat->fileType = FILE_TYPE_SYMLINK; } else if (fileTagInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { @@ -524,12 +532,21 @@ Java_net_rubygrapefruit_platform_internal_jni_FileEventFunctions_closeWatch(JNIE JNIEXPORT void JNICALL Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_stat(JNIEnv *env, jclass target, jstring path, jboolean followLink, jobject dest, jobject result) { +#ifdef WINDOWS_MIN jclass destClass = env->GetObjectClass(dest); jmethodID mid = env->GetMethodID(destClass, "details", "(IJJ)V"); if (mid == NULL) { mark_failed_with_message(env, "could not find method", result); return; } +#else + jclass destClass = env->GetObjectClass(dest); + jmethodID mid = env->GetMethodID(destClass, "details", "(IJJIJ)V"); + if (mid == NULL) { + mark_failed_with_message(env, "could not find method", result); + return; + } +#endif wchar_t* pathStr = java_to_wchar_path(env, path, result); file_stat_t fileStat; @@ -539,7 +556,12 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_stat(JNIEnv * mark_failed_with_code(env, "could not file attributes", errorCode, NULL, result); return; } + +#ifdef WINDOWS_MIN env->CallVoidMethod(dest, mid, fileStat.fileType, fileStat.size, fileStat.lastModified); +#else + env->CallVoidMethod(dest, mid, fileStat.fileType, fileStat.size, fileStat.lastModified, fileStat.volumeId, fileStat.fileId); +#endif } JNIEXPORT void JNICALL @@ -587,6 +609,8 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_readdir(JNIEn FILE_TYPE_FILE; fileInfo.lastModified = lastModifiedNanos(&entry.ftLastWriteTime); fileInfo.size = ((jlong)entry.nFileSizeHigh << 32) | entry.nFileSizeLow; + fileInfo.volumeId = 0; + fileInfo.fileId = 0; } // Add entry @@ -619,6 +643,7 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirIs typedef struct fast_readdir_handle { HANDLE handle; wchar_t* pathStr; + ULONG volumeSerialNumber; } readdir_fast_handle_t; #endif @@ -666,6 +691,17 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirOp free(pathStr); return NULL; } + + // This call allows retrieving the volume ID of this directory (and all its entries) + BY_HANDLE_FILE_INFORMATION fileInfo; + BOOL ok = GetFileInformationByHandle(handle, &fileInfo); + if (!ok) { + mark_failed_with_errno(env, "could not open directory", result); + free(pathStr); + CloseHandle(handle); + return NULL; + } + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)LocalAlloc(LPTR, sizeof(readdir_fast_handle_t)); if (readdirHandle == NULL) { mark_failed_with_code(env, "Out of native memory", ERROR_OUTOFMEMORY, NULL, result); @@ -675,6 +711,7 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirOp } readdirHandle->handle = handle; readdirHandle->pathStr = pathStr; + readdirHandle->volumeSerialNumber = fileInfo.dwVolumeSerialNumber; return (jlong)readdirHandle; #endif } @@ -694,6 +731,20 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirCl #endif } +// +// Returns the volume id of the directory opened by fastReaddirOpen +// +JNIEXPORT jint JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirGetVolumeId(JNIEnv *env, jclass target, jlong handle, jobject result) { +#ifdef WINDOWS_MIN + mark_failed_with_code(env, "Operation not supported", ERROR_CALL_NOT_IMPLEMENTED, NULL, result); + return 0; +#else + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)handle; + return readdirHandle->volumeSerialNumber; +#endif +} + // // Reads the next batch of entries from the directory. // Returns JNI_TRUE on success and if there are more entries found diff --git a/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java b/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java index 115fe750..f30048e1 100644 --- a/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java +++ b/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java @@ -44,4 +44,12 @@ enum Type { * Returns the last modification time of this file, in ms since epoch. Returns 0 when this file does not exist. */ long getLastModifiedTime(); + + /** + * Returns an object that uniquely identifies the given file, or null if a file key is not available. + * + *

See BasicFileAttributes.fileKey() + * for a more in depth explanation.

+ */ + Object getKey(); } diff --git a/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java b/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java index fcbedfae..1c75b432 100644 --- a/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java +++ b/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java @@ -25,4 +25,13 @@ */ @ThreadSafe public interface WindowsFileInfo extends FileInfo { + /** + * Returns the volume ID (serial number) of the file. + */ + int getVolumeId(); + + /** + * Returns the file ID of the file, unique within the volume identified by {@link #getVolumeId()}. + */ + long getFileId(); } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java index 0f4e9016..f957b1ee 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java @@ -86,6 +86,7 @@ private class FastLister implements DirectoryLister { private static final int OFFSETOF_FILE_ATTRIBUTES = 56; private static final int OFFSETOF_FILENAME_LENGTH = 60; private static final int OFFSETOF_EA_SIZE = 64; + private static final int OFFSETOF_FILE_ID = 72; private static final int OFFSETOF_FILENAME = 80; /** @@ -109,6 +110,10 @@ public List listDir(File dir, boolean linkTarget) throws Nat if (result.isFailed()) { throw listDirFailure(dir, result); } + int volumeId = WindowsFileFunctions.fastReaddirGetVolumeId(handle, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } try { NtQueryDirectoryFileContext context = getNtQueryDirectoryFileContext(); @@ -120,7 +125,7 @@ public List listDir(File dir, boolean linkTarget) throws Nat int entryOffset = 0; while (true) { // Read entry from buffer - entryOffset = addFullDirEntry(context, dir, linkTarget, entryOffset, dirList); + entryOffset = addFullDirEntry(context, dir, linkTarget, volumeId, entryOffset, dirList); // If we reached end of buffer, fetch next set of entries if (entryOffset == 0) { @@ -148,7 +153,7 @@ public List listDir(File dir, boolean linkTarget) throws Nat *

Returns the byte offset of the next entry in {@link NtQueryDirectoryFileContext#buffer} if there is one, * or {@code 0} if there is no next entry.

*/ - private int addFullDirEntry(NtQueryDirectoryFileContext context, File dir, boolean followLink, int entryOffset, WindowsDirList dirList) { + private int addFullDirEntry(NtQueryDirectoryFileContext context, File dir, boolean followLink, int volumeId, int entryOffset, WindowsDirList dirList) { // typedef struct _FILE_ID_FULL_DIR_INFORMATION { // ULONG NextEntryOffset; // offset = 0 // ULONG FileIndex; // offset = 4 @@ -175,6 +180,7 @@ private int addFullDirEntry(NtQueryDirectoryFileContext context, File dir, boole // int fileAttributes = context.buffer.getInt(entryOffset + OFFSETOF_FILE_ATTRIBUTES); int reparseTagData = context.buffer.getInt(entryOffset + OFFSETOF_EA_SIZE); + long fileId = context.buffer.getLong(entryOffset + OFFSETOF_FILE_ID); FileInfo.Type type = getFileType(fileAttributes, reparseTagData); @@ -189,7 +195,7 @@ private int addFullDirEntry(NtQueryDirectoryFileContext context, File dir, boole WindowsFileInfo targetInfo = stat(new File(dir, fileName), true); dirList.addFile(fileName, targetInfo); } else { - dirList.addFile(fileName, type.ordinal(), fileSize, lastModified); + dirList.addFile(fileName, type, fileSize, WindowsFileTime.toJavaTime(lastModified), volumeId, fileId); } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/DirList.java b/src/main/java/net/rubygrapefruit/platform/internal/DirList.java index f36e5509..491a5b78 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/DirList.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/DirList.java @@ -28,11 +28,19 @@ public class DirList { // Called from native code @SuppressWarnings("UnusedDeclaration") public void addFile(String name, int type, long size, long lastModified) { - DefaultDirEntry fileStat = new DefaultDirEntry(name, FileInfo.Type.values()[type], size, lastModified); - files.add(fileStat); + addFile(name, FileInfo.Type.values()[type], size, lastModified); } - private static class DefaultDirEntry implements DirEntry { + void addFile(String name, FileInfo.Type type, long size, long lastModified) { + DefaultDirEntry fileStat = new DefaultDirEntry(name, type, size, lastModified); + addEntry(fileStat); + } + + void addEntry(DirEntry entry) { + files.add(entry); + } + + protected static class DefaultDirEntry implements DirEntry { private final String name; private final Type type; private final long size; @@ -65,5 +73,9 @@ public long getLastModifiedTime() { public long getSize() { return size; } + + public Object getKey() { + return null; + } } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java b/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java index 52ea3025..92fa1e1c 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java @@ -74,4 +74,8 @@ public long getBlockSize() { public long getLastModifiedTime() { return modificationTime; } + + public Object getKey() { + return null; + } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java index 0054c925..603a5034 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java @@ -16,6 +16,7 @@ package net.rubygrapefruit.platform.internal; +import net.rubygrapefruit.platform.file.FileInfo; import net.rubygrapefruit.platform.file.WindowsFileInfo; public class WindowsDirList extends DirList { @@ -23,10 +24,46 @@ public class WindowsDirList extends DirList { @SuppressWarnings("UnusedDeclaration") @Override public void addFile(String name, int type, long size, long lastModified) { - super.addFile(name, type, size, WindowsFileTime.toJavaTime(lastModified)); + addFile(name, FileInfo.Type.values()[type], size, WindowsFileTime.toJavaTime(lastModified), 0, 0); } - public void addFile(String name, WindowsFileInfo fileInfo) { - super.addFile(name, fileInfo.getType().ordinal(), fileInfo.getSize(), fileInfo.getLastModifiedTime()); + void addFile(String name, WindowsFileInfo fileInfo) { + addFile(name, fileInfo.getType(), fileInfo.getSize(), fileInfo.getLastModifiedTime(), fileInfo.getVolumeId(), fileInfo.getFileId()); + } + + void addFile(String name, FileInfo.Type type, long size, long lastModified, int volumeId, long fileId) { + if (volumeId == 0 && fileId == 0) { + super.addFile(name, type, size, lastModified); + } else { + WindowsDirListEntry entry = new WindowsDirListEntry(name, type, size, lastModified, volumeId, fileId); + addEntry(entry); + } + } + + protected static class WindowsDirListEntry extends DefaultDirEntry { + private final int volumeId; + private final long fileId; + // Lazily initialized to avoid extra allocation if not needed + private volatile WindowsFileKey key; + + WindowsDirListEntry(String name, Type type, long size, long lastModified, int volumeId, long fileId) { + super(name, type, size, lastModified); + this.volumeId = volumeId; + this.fileId = fileId; + } + + public Object getKey() { + if (volumeId == 0 && fileId == 0) { + return null; + } + if (key == null) { + synchronized (this) { + if (key == null) { + key = new WindowsFileKey(volumeId, fileId); + } + } + } + return key; + } } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileKey.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileKey.java new file mode 100644 index 00000000..43aa14d8 --- /dev/null +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileKey.java @@ -0,0 +1,39 @@ +package net.rubygrapefruit.platform.internal; + +final class WindowsFileKey { + private final int volumeId; + private final long fileId; + + WindowsFileKey(int volumeId, long fileId) { + this.volumeId = volumeId; + this.fileId = fileId; + } + + @Override + public int hashCode() { + return (int)(volumeId * 31 + fileId); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) + return true; + if (!(obj instanceof WindowsFileKey)) + return false; + + WindowsFileKey other = (WindowsFileKey) obj; + return (this.volumeId == other.volumeId) && + (this.fileId == other.fileId); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("(volumeId=") + .append(Integer.toHexString(volumeId)) + .append(",fileId=") + .append(Long.toHexString(fileId)) + .append(')'); + return sb.toString(); + } +} diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java index 5e85b99d..8bed8067 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java @@ -23,6 +23,10 @@ public class WindowsFileStat implements WindowsFileInfo { private Type type; private long size; private long lastModified; + private int volumeId; + private long fileId; + // Lazily initialized to avoid extra allocation if not needed + private volatile WindowsFileKey key; public WindowsFileStat(String path) { this.path = path; @@ -34,6 +38,14 @@ public void details(int type, long size, long lastModifiedWinTime) { this.lastModified = WindowsFileTime.toJavaTime(lastModifiedWinTime); } + public void details(int type, long size, long lastModifiedWinTime, int volumeId, long fileId) { + this.type = Type.values()[type]; + this.size = size; + this.lastModified = this.type == Type.Missing ? 0 : WindowsFileTime.toJavaTime(lastModifiedWinTime); + this.volumeId = volumeId; + this.fileId = fileId; + } + @Override public String toString() { return path; @@ -50,4 +62,26 @@ public long getSize() { public long getLastModifiedTime() { return lastModified; } + + public int getVolumeId() { + return volumeId; + } + + public long getFileId() { + return fileId; + } + + public Object getKey() { + if (volumeId == 0 && fileId == 0) { + return null; + } + if (key == null) { + synchronized (this) { + if (key == null) { + key = new WindowsFileKey(volumeId, fileId); + } + } + } + return key; + } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java b/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java index 1e75e99d..ea142ce5 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java @@ -31,5 +31,6 @@ public class WindowsFileFunctions { public static native boolean fastReaddirIsSupported(); public static native long fastReaddirOpen(String path, FunctionResult result); public static native void fastReaddirClose(long handle); + public static native int fastReaddirGetVolumeId(long handle, FunctionResult result); public static native boolean fastReaddirNext(long handle, ByteBuffer buffer, FunctionResult result); } diff --git a/src/shared/headers/generic.h b/src/shared/headers/generic.h index 4722ee5e..08ff5d32 100755 --- a/src/shared/headers/generic.h +++ b/src/shared/headers/generic.h @@ -110,6 +110,8 @@ typedef struct file_stat { jint fileType; jlong lastModified; jlong size; + jint volumeId; + jlong fileId; } file_stat_t; #ifdef __cplusplus diff --git a/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java b/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java index 5a7137ea..831d71a2 100755 --- a/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java +++ b/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java @@ -404,6 +404,9 @@ private static void ls(String path, boolean followLinks) { System.out.println(); System.out.println("* Name: " + entry.getName()); System.out.println("* Type: " + entry.getType()); + if (entry.getKey() != null) { + System.out.println("* Key: " + entry.getKey()); + } stat(new File(dir, entry.getName()), entry); } } @@ -424,6 +427,9 @@ private static void stat(String path, boolean linkTarget) { System.out.println(); System.out.println("* File: " + file); System.out.println("* Type: " + stat.getType()); + if (stat.getKey() != null) { + System.out.println("* Key: " + stat.getKey()); + } if (stat.getType() != FileInfo.Type.Missing) { if (stat instanceof PosixFileInfo) { stat(file, (PosixFileInfo) stat); From dddc65f28484c99852afec79e6161d26997fc689 Mon Sep 17 00:00:00 2001 From: Renaud Paquay Date: Wed, 7 Aug 2019 15:48:55 -0700 Subject: [PATCH 4/4] Add file key support on Posix platforms Also updated tests, since all platforms now include support for file keys --- src/main/cpp/posix.cpp | 17 +++++---- .../platform/internal/DirList.java | 35 +++++++++++++----- .../{WindowsFileKey.java => FileKey.java} | 9 ++--- .../platform/internal/FileStat.java | 29 +++++++++++++-- .../platform/internal/WindowsDirList.java | 36 ------------------- .../platform/internal/WindowsFileStat.java | 12 +++---- .../platform/file/FilesTest.groovy | 10 ++++++ 7 files changed, 85 insertions(+), 63 deletions(-) rename src/main/java/net/rubygrapefruit/platform/internal/{WindowsFileKey.java => FileKey.java} (82%) diff --git a/src/main/cpp/posix.cpp b/src/main/cpp/posix.cpp index 56db0d0a..21a88e2a 100755 --- a/src/main/cpp/posix.cpp +++ b/src/main/cpp/posix.cpp @@ -105,14 +105,14 @@ void unpackStat(struct stat* source, file_stat_t* result) { #else result->lastModified = toMillis(source->st_mtimespec); #endif - result->volumeId = 0; - result->fileId = 0; + result->volumeId = (jint)source->st_dev; + result->fileId = (jlong)source->st_ino; } JNIEXPORT void JNICALL Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_stat(JNIEnv *env, jclass target, jstring path, jboolean followLink, jobject dest, jobject result) { jclass destClass = env->GetObjectClass(dest); - jmethodID mid = env->GetMethodID(destClass, "details", "(IIIIJJI)V"); + jmethodID mid = env->GetMethodID(destClass, "details", "(IIIIJJIIJ)V"); if (mid == NULL) { mark_failed_with_message(env, "could not find method", result); return; @@ -136,7 +136,7 @@ Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_stat(JNIEnv *en } if (retval != 0) { - env->CallVoidMethod(dest, mid, FILE_TYPE_MISSING, (jint)0, (jint)0, (jint)0, (jlong)0, (jlong)0, (jint)0); + env->CallVoidMethod(dest, mid, FILE_TYPE_MISSING, (jint)0, (jint)0, (jint)0, (jlong)0, (jlong)0, (jint)0, (jint)0, (jlong)0); } else { file_stat_t fileResult; unpackStat(&fileInfo, &fileResult); @@ -148,14 +148,16 @@ Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_stat(JNIEnv *en (jint)fileInfo.st_gid, fileResult.size, fileResult.lastModified, - (jint)fileInfo.st_blksize); + (jint)fileInfo.st_blksize, + fileResult.volumeId, + fileResult.fileId); } } JNIEXPORT void JNICALL Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_readdir(JNIEnv *env, jclass target, jstring path, jboolean followLink, jobject contents, jobject result) { jclass contentsClass = env->GetObjectClass(contents); - jmethodID mid = env->GetMethodID(contentsClass, "addFile", "(Ljava/lang/String;IJJ)V"); + jmethodID mid = env->GetMethodID(contentsClass, "addFile", "(Ljava/lang/String;IJJIJ)V"); if (mid == NULL) { mark_failed_with_message(env, "could not find method", result); return; @@ -216,7 +218,8 @@ Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_readdir(JNIEnv } jstring childName = char_to_java(env, entry.d_name, result); - env->CallVoidMethod(contents, mid, childName, fileResult.fileType, fileResult.size, fileResult.lastModified); + env->CallVoidMethod(contents, mid, childName, fileResult.fileType, fileResult.size, fileResult.lastModified, + fileResult.volumeId, fileResult.fileId); } closedir(dir); diff --git a/src/main/java/net/rubygrapefruit/platform/internal/DirList.java b/src/main/java/net/rubygrapefruit/platform/internal/DirList.java index 491a5b78..5afbc292 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/DirList.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/DirList.java @@ -28,16 +28,18 @@ public class DirList { // Called from native code @SuppressWarnings("UnusedDeclaration") public void addFile(String name, int type, long size, long lastModified) { - addFile(name, FileInfo.Type.values()[type], size, lastModified); + addFile(name, type, size, lastModified, 0 ,0); } - void addFile(String name, FileInfo.Type type, long size, long lastModified) { - DefaultDirEntry fileStat = new DefaultDirEntry(name, type, size, lastModified); - addEntry(fileStat); + // Called from native code + @SuppressWarnings("UnusedDeclaration") + public void addFile(String name, int type, long size, long lastModified, int volumeId, long fileId) { + addFile(name, FileInfo.Type.values()[type], size, lastModified, volumeId, fileId); } - void addEntry(DirEntry entry) { - files.add(entry); + void addFile(String name, FileInfo.Type type, long size, long lastModified, int volumeId, long fileId) { + DefaultDirEntry fileStat = new DefaultDirEntry(name, type, size, lastModified, volumeId, fileId); + files.add(fileStat); } protected static class DefaultDirEntry implements DirEntry { @@ -45,12 +47,18 @@ protected static class DefaultDirEntry implements DirEntry { private final Type type; private final long size; private final long lastModified; + private final int volumeId; + private final long fileId; + // Lazily initialized to avoid extra allocation if not needed + private volatile FileKey key; - DefaultDirEntry(String name, Type type, long size, long lastModified) { + DefaultDirEntry(String name, Type type, long size, long lastModified, int volumeId, long fileId) { this.name = name; this.type = type; this.size = size; this.lastModified = lastModified; + this.volumeId = volumeId; + this.fileId = fileId; } @Override @@ -75,7 +83,18 @@ public long getSize() { } public Object getKey() { - return null; + if (volumeId == 0 && fileId == 0) { + return null; + } + if (key == null) { + synchronized (this) { + if (key == null) { + key = new FileKey(volumeId, fileId); + } + } + } + return key; } + } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileKey.java b/src/main/java/net/rubygrapefruit/platform/internal/FileKey.java similarity index 82% rename from src/main/java/net/rubygrapefruit/platform/internal/WindowsFileKey.java rename to src/main/java/net/rubygrapefruit/platform/internal/FileKey.java index 43aa14d8..aa9a8b0c 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileKey.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/FileKey.java @@ -1,10 +1,10 @@ package net.rubygrapefruit.platform.internal; -final class WindowsFileKey { +final class FileKey { private final int volumeId; private final long fileId; - WindowsFileKey(int volumeId, long fileId) { + FileKey(int volumeId, long fileId) { this.volumeId = volumeId; this.fileId = fileId; } @@ -18,10 +18,10 @@ public int hashCode() { public boolean equals(Object obj) { if (obj == this) return true; - if (!(obj instanceof WindowsFileKey)) + if (!(obj instanceof FileKey)) return false; - WindowsFileKey other = (WindowsFileKey) obj; + FileKey other = (FileKey) obj; return (this.volumeId == other.volumeId) && (this.fileId == other.fileId); } @@ -36,4 +36,5 @@ public String toString() { .append(')'); return sb.toString(); } + } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java b/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java index 92fa1e1c..fd6384b1 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java @@ -27,12 +27,17 @@ public class FileStat implements PosixFileInfo { private long size; private long modificationTime; private long blockSize; + private int volumeId; + private long fileId; + // Lazily initialized to avoid extra allocation if not needed + private volatile FileKey key; public FileStat(String path) { this.path = path; } - public void details(int type, int mode, int uid, int gid, long size, long modificationTime, int blockSize) { + // Called from native code + public void details(int type, int mode, int uid, int gid, long size, long modificationTime, int blockSize, int volumeId, long fileId) { this.type = Type.values()[type]; this.mode = mode; this.uid = uid; @@ -40,6 +45,8 @@ public void details(int type, int mode, int uid, int gid, long size, long modifi this.size = size; this.modificationTime = modificationTime; this.blockSize = blockSize; + this.volumeId = volumeId; + this.fileId = fileId; } @Override @@ -76,6 +83,24 @@ public long getLastModifiedTime() { } public Object getKey() { - return null; + if (volumeId == 0 && fileId == 0) { + return null; + } + if (key == null) { + synchronized (this) { + if (key == null) { + key = new FileKey(volumeId, fileId); + } + } + } + return key; + } + + public int getVolumeId() { + return volumeId; + } + + public long getFileId() { + return fileId; } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java index 603a5034..2588373f 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java @@ -30,40 +30,4 @@ public void addFile(String name, int type, long size, long lastModified) { void addFile(String name, WindowsFileInfo fileInfo) { addFile(name, fileInfo.getType(), fileInfo.getSize(), fileInfo.getLastModifiedTime(), fileInfo.getVolumeId(), fileInfo.getFileId()); } - - void addFile(String name, FileInfo.Type type, long size, long lastModified, int volumeId, long fileId) { - if (volumeId == 0 && fileId == 0) { - super.addFile(name, type, size, lastModified); - } else { - WindowsDirListEntry entry = new WindowsDirListEntry(name, type, size, lastModified, volumeId, fileId); - addEntry(entry); - } - } - - protected static class WindowsDirListEntry extends DefaultDirEntry { - private final int volumeId; - private final long fileId; - // Lazily initialized to avoid extra allocation if not needed - private volatile WindowsFileKey key; - - WindowsDirListEntry(String name, Type type, long size, long lastModified, int volumeId, long fileId) { - super(name, type, size, lastModified); - this.volumeId = volumeId; - this.fileId = fileId; - } - - public Object getKey() { - if (volumeId == 0 && fileId == 0) { - return null; - } - if (key == null) { - synchronized (this) { - if (key == null) { - key = new WindowsFileKey(volumeId, fileId); - } - } - } - return key; - } - } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java index 8bed8067..684bcf4f 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java @@ -26,22 +26,22 @@ public class WindowsFileStat implements WindowsFileInfo { private int volumeId; private long fileId; // Lazily initialized to avoid extra allocation if not needed - private volatile WindowsFileKey key; + private volatile FileKey key; public WindowsFileStat(String path) { this.path = path; } + // Called from native code public void details(int type, long size, long lastModifiedWinTime) { - this.type = Type.values()[type]; - this.size = size; - this.lastModified = WindowsFileTime.toJavaTime(lastModifiedWinTime); + details(type, size,lastModifiedWinTime, 0, 0); } + // Called from native code public void details(int type, long size, long lastModifiedWinTime, int volumeId, long fileId) { this.type = Type.values()[type]; this.size = size; - this.lastModified = this.type == Type.Missing ? 0 : WindowsFileTime.toJavaTime(lastModifiedWinTime); + this.lastModified = WindowsFileTime.toJavaTime(lastModifiedWinTime); this.volumeId = volumeId; this.fileId = fileId; } @@ -78,7 +78,7 @@ public Object getKey() { if (key == null) { synchronized (this) { if (key == null) { - key = new WindowsFileKey(volumeId, fileId); + key = new FileKey(volumeId, fileId); } } } diff --git a/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy b/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy index c3232fae..ceb54e32 100755 --- a/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy +++ b/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy @@ -41,6 +41,7 @@ abstract class FilesTest extends AbstractFilesTest { void assertIsFile(FileInfo stat, File file) { assert stat.type == FileInfo.Type.File assert stat.size == file.length() + assert stat.key != null def attributes = attributes(file) assertTimestampMatches(stat.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(stat.lastModifiedTime, file.lastModified()) @@ -50,6 +51,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.type == FileInfo.Type.File assert entry.name == name assert entry.size == file.length() + assert entry.key != null def attributes = attributes(file) assertTimestampMatches(entry.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(entry.lastModifiedTime, file.lastModified()) @@ -58,6 +60,7 @@ abstract class FilesTest extends AbstractFilesTest { void assertIsDirectory(FileInfo stat, File file) { assert stat.type == FileInfo.Type.Directory assert stat.size == 0 + assert stat.key != null def attributes = attributes(file) assertTimestampMatches(stat.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(stat.lastModifiedTime, file.lastModified()) @@ -67,6 +70,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.type == FileInfo.Type.Directory assert entry.name == file.name assert entry.size == 0 + assert entry.key != null def attributes = attributes(file) assertTimestampMatches(entry.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(entry.lastModifiedTime, file.lastModified()) @@ -75,6 +79,7 @@ abstract class FilesTest extends AbstractFilesTest { void assertIsSymlink(FileInfo stat, File file) { assert stat.type == FileInfo.Type.Symlink assert stat.size == 0 + assert stat.key != null def attributes = attributes(file) assertTimestampMatches(stat.lastModifiedTime, attributes.lastModifiedTime().toMillis()) // Can't check `file.lastModified()` as it follows symlinks @@ -84,6 +89,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.type == FileInfo.Type.Symlink assert entry.name == file.name assert entry.size == 0 + assert entry.key != null def attributes = attributes(file) assertTimestampMatches(entry.lastModifiedTime, attributes.lastModifiedTime().toMillis()) // Can't check `file.lastModified()` as it follows symlinks @@ -93,6 +99,7 @@ abstract class FilesTest extends AbstractFilesTest { assert stat.type == FileInfo.Type.Missing assert stat.size == 0 assert stat.lastModifiedTime == 0 + assert stat.key == null } void assertIsMissing(DirEntry entry, String name) { @@ -100,6 +107,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.name == name assert entry.size == 0 assert entry.lastModifiedTime == 0 + assert entry.key == null } def "caches file instance"() { @@ -495,6 +503,7 @@ abstract class FilesTest extends AbstractFilesTest { def linkEntry = files[2] assertIsSymlink(linkEntry, childLink) + assert linkEntry.key != fileEntry.key def missingEntry = files[3] assertIsSymlink(missingEntry, childMissingLink) @@ -541,6 +550,7 @@ abstract class FilesTest extends AbstractFilesTest { def linkEntry = files[2] assertIsFile(linkEntry, childFile, childLink.name) + assert linkEntry.key == fileEntry.key def missingEntry = files[3] assertIsMissing(missingEntry, childMissingLink.name)