diff --git a/build.gradle b/build.gradle index 28d8464e..7b99f285 100755 --- a/build.gradle +++ b/build.gradle @@ -274,6 +274,9 @@ model { } else if (targetPlatform.operatingSystem.windows) { if (name.contains("_min")) { cppCompiler.define "WINDOWS_MIN" + } else { + // For NtQueryDirectoryFile + linker.args "ntdll.lib" } cppCompiler.args "-I${org.gradle.internal.jvm.Jvm.current().javaHome}/include" cppCompiler.args "-I${org.gradle.internal.jvm.Jvm.current().javaHome}/include/win32" diff --git a/src/main/cpp/posix.cpp b/src/main/cpp/posix.cpp index 8682b809..21a88e2a 100755 --- a/src/main/cpp/posix.cpp +++ b/src/main/cpp/posix.cpp @@ -105,12 +105,14 @@ void unpackStat(struct stat* source, file_stat_t* result) { #else result->lastModified = toMillis(source->st_mtimespec); #endif + result->volumeId = (jint)source->st_dev; + result->fileId = (jlong)source->st_ino; } JNIEXPORT void JNICALL Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_stat(JNIEnv *env, jclass target, jstring path, jboolean followLink, jobject dest, jobject result) { jclass destClass = env->GetObjectClass(dest); - jmethodID mid = env->GetMethodID(destClass, "details", "(IIIIJJI)V"); + jmethodID mid = env->GetMethodID(destClass, "details", "(IIIIJJIIJ)V"); if (mid == NULL) { mark_failed_with_message(env, "could not find method", result); return; @@ -134,7 +136,7 @@ Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_stat(JNIEnv *en } if (retval != 0) { - env->CallVoidMethod(dest, mid, FILE_TYPE_MISSING, (jint)0, (jint)0, (jint)0, (jlong)0, (jlong)0, (jint)0); + env->CallVoidMethod(dest, mid, FILE_TYPE_MISSING, (jint)0, (jint)0, (jint)0, (jlong)0, (jlong)0, (jint)0, (jint)0, (jlong)0); } else { file_stat_t fileResult; unpackStat(&fileInfo, &fileResult); @@ -146,14 +148,16 @@ Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_stat(JNIEnv *en (jint)fileInfo.st_gid, fileResult.size, fileResult.lastModified, - (jint)fileInfo.st_blksize); + (jint)fileInfo.st_blksize, + fileResult.volumeId, + fileResult.fileId); } } JNIEXPORT void JNICALL Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_readdir(JNIEnv *env, jclass target, jstring path, jboolean followLink, jobject contents, jobject result) { jclass contentsClass = env->GetObjectClass(contents); - jmethodID mid = env->GetMethodID(contentsClass, "addFile", "(Ljava/lang/String;IJJ)V"); + jmethodID mid = env->GetMethodID(contentsClass, "addFile", "(Ljava/lang/String;IJJIJ)V"); if (mid == NULL) { mark_failed_with_message(env, "could not find method", result); return; @@ -207,12 +211,15 @@ Java_net_rubygrapefruit_platform_internal_jni_PosixFileFunctions_readdir(JNIEnv fileResult.fileType = FILE_TYPE_MISSING; fileResult.size = 0; fileResult.lastModified = 0; + fileResult.volumeId = 0; + fileResult.fileId = 0; } else { unpackStat(&fileInfo, &fileResult); } jstring childName = char_to_java(env, entry.d_name, result); - env->CallVoidMethod(contents, mid, childName, fileResult.fileType, fileResult.size, fileResult.lastModified); + env->CallVoidMethod(contents, mid, childName, fileResult.fileType, fileResult.size, fileResult.lastModified, + fileResult.volumeId, fileResult.fileId); } closedir(dir); diff --git a/src/main/cpp/win.cpp b/src/main/cpp/win.cpp index df0a1ac5..1e8d0601 100755 --- a/src/main/cpp/win.cpp +++ b/src/main/cpp/win.cpp @@ -21,6 +21,9 @@ #include #include #include +#ifndef WINDOWS_MIN +#include "ntifs_min.h" +#endif #define ALL_COLORS (FOREGROUND_BLUE|FOREGROUND_RED|FOREGROUND_GREEN) @@ -31,8 +34,18 @@ void mark_failed_with_errno(JNIEnv *env, const char* message, jobject result) { mark_failed_with_code(env, message, GetLastError(), NULL, result); } +#ifndef WINDOWS_MIN +/* + * Marks the given result as failed, using a NTSTATUS error code + */ +void mark_failed_with_ntstatus(JNIEnv *env, const char* message, NTSTATUS status, jobject result) { + ULONG win32ErrorCode = RtlNtStatusToDosError(status); + mark_failed_with_code(env, message, win32ErrorCode, NULL, result); +} +#endif + int map_error_code(int error_code) { - if (error_code == ERROR_PATH_NOT_FOUND) { + if (error_code == ERROR_FILE_NOT_FOUND || error_code == ERROR_PATH_NOT_FOUND) { return FAILURE_NO_SUCH_FILE; } if (error_code == ERROR_DIRECTORY) { @@ -81,6 +94,7 @@ bool is_path_absolute_unc(wchar_t* path, int path_len) { // // Returns a UTF-16 string that is the concatenation of |prefix| and |path|. +// The string must be deallocated with a call to free(). // wchar_t* add_prefix(wchar_t* path, int path_len, wchar_t* prefix) { int prefix_len = wcslen(prefix); @@ -156,10 +170,6 @@ jlong lastModifiedNanos(FILETIME* time) { return ((jlong)time->dwHighDateTime << 32) | time->dwLowDateTime; } -jlong lastModifiedNanos(LARGE_INTEGER* time) { - return ((jlong)time->HighPart << 32) | time->LowPart; -} - // // Retrieves the file attributes for the file specified by |pathStr|. // If |followLink| is true, symbolic link targets are resolved. @@ -179,11 +189,15 @@ DWORD get_file_stat(wchar_t* pathStr, jboolean followLink, file_stat_t* pFileSta pFileStat->lastModified = 0; pFileStat->size = 0; pFileStat->fileType = FILE_TYPE_MISSING; + pFileStat->volumeId = 0; + pFileStat->fileId = 0; return ERROR_SUCCESS; } return error; } pFileStat->lastModified = lastModifiedNanos(&attr.ftLastWriteTime); + pFileStat->volumeId = 0; + pFileStat->fileId = 0; if (attr.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { pFileStat->size = 0; pFileStat->fileType = FILE_TYPE_DIRECTORY; @@ -213,6 +227,8 @@ DWORD get_file_stat(wchar_t* pathStr, jboolean followLink, file_stat_t* pFileSta pFileStat->lastModified = 0; pFileStat->size = 0; pFileStat->fileType = FILE_TYPE_MISSING; + pFileStat->volumeId = 0; + pFileStat->fileId = 0; return ERROR_SUCCESS; } return error; @@ -240,6 +256,8 @@ DWORD get_file_stat(wchar_t* pathStr, jboolean followLink, file_stat_t* pFileSta pFileStat->lastModified = lastModifiedNanos(&fileInfo.ftLastWriteTime); pFileStat->size = 0; + pFileStat->volumeId = fileInfo.dwVolumeSerialNumber; + pFileStat->fileId = ((LONGLONG)fileInfo.nFileIndexHigh << 32) | fileInfo.nFileIndexLow; if (is_file_symlink(fileTagInfo.FileAttributes, fileTagInfo.ReparseTag)) { pFileStat->fileType = FILE_TYPE_SYMLINK; } else if (fileTagInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { @@ -514,12 +532,21 @@ Java_net_rubygrapefruit_platform_internal_jni_FileEventFunctions_closeWatch(JNIE JNIEXPORT void JNICALL Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_stat(JNIEnv *env, jclass target, jstring path, jboolean followLink, jobject dest, jobject result) { +#ifdef WINDOWS_MIN jclass destClass = env->GetObjectClass(dest); jmethodID mid = env->GetMethodID(destClass, "details", "(IJJ)V"); if (mid == NULL) { mark_failed_with_message(env, "could not find method", result); return; } +#else + jclass destClass = env->GetObjectClass(dest); + jmethodID mid = env->GetMethodID(destClass, "details", "(IJJIJ)V"); + if (mid == NULL) { + mark_failed_with_message(env, "could not find method", result); + return; + } +#endif wchar_t* pathStr = java_to_wchar_path(env, path, result); file_stat_t fileStat; @@ -529,7 +556,12 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_stat(JNIEnv * mark_failed_with_code(env, "could not file attributes", errorCode, NULL, result); return; } + +#ifdef WINDOWS_MIN env->CallVoidMethod(dest, mid, fileStat.fileType, fileStat.size, fileStat.lastModified); +#else + env->CallVoidMethod(dest, mid, fileStat.fileType, fileStat.size, fileStat.lastModified, fileStat.volumeId, fileStat.fileId); +#endif } JNIEXPORT void JNICALL @@ -577,6 +609,8 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_readdir(JNIEn FILE_TYPE_FILE; fileInfo.lastModified = lastModifiedNanos(&entry.ftLastWriteTime); fileInfo.size = ((jlong)entry.nFileSizeHigh << 32) | entry.nFileSizeLow; + fileInfo.volumeId = 0; + fileInfo.fileId = 0; } // Add entry @@ -593,6 +627,168 @@ Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_readdir(JNIEn FindClose(dirHandle); } +// +// Returns "true" if the various fastReaddirXxx calls are supported on this platform. +// +JNIEXPORT jboolean JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirIsSupported(JNIEnv *env, jclass target) { +#ifdef WINDOWS_MIN + return JNI_FALSE; +#else + return JNI_TRUE; +#endif +} + +#ifndef WINDOWS_MIN +typedef struct fast_readdir_handle { + HANDLE handle; + wchar_t* pathStr; + ULONG volumeSerialNumber; +} readdir_fast_handle_t; +#endif + +#ifndef WINDOWS_MIN +NTSTATUS invokeNtQueryDirectoryFile(HANDLE handle, BYTE* buffer, ULONG bufferSize) { + IO_STATUS_BLOCK ioStatusBlock; + + return NtQueryDirectoryFile( + handle, // FileHandle + NULL, // Event + NULL, // ApcRoutine + NULL, // ApcContext + &ioStatusBlock, // IoStatusBlock + buffer, // FileInformation + bufferSize, // Length + FileIdFullDirectoryInformation, // FileInformationClass + FALSE, // ReturnSingleEntry + NULL, // FileName + FALSE); // RestartScan +} +#endif + +// +// Opens a directory for file enumeration and returns a handle to a |fast_readdir_handle| structure +// on success. The handle must be released by calling "xxx_fastReaddirClose" when done. +// Returns NULL on failure (and sets error message in |result|). +// +JNIEXPORT jlong JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirOpen(JNIEnv *env, jclass target, jstring path, jobject result) { +#ifdef WINDOWS_MIN + mark_failed_with_code(env, "Operation not supported", ERROR_CALL_NOT_IMPLEMENTED, NULL, result); + return NULL; +#else + // Open file for directory listing + wchar_t* pathStr = java_to_wchar_path(env, path, result); + if (pathStr == NULL) { + mark_failed_with_code(env, "Out of native memory", ERROR_OUTOFMEMORY, NULL, result); + return NULL; + } + HANDLE handle = CreateFileW(pathStr, FILE_LIST_DIRECTORY, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (handle == INVALID_HANDLE_VALUE) { + mark_failed_with_errno(env, "could not open directory", result); + free(pathStr); + return NULL; + } + + // This call allows retrieving the volume ID of this directory (and all its entries) + BY_HANDLE_FILE_INFORMATION fileInfo; + BOOL ok = GetFileInformationByHandle(handle, &fileInfo); + if (!ok) { + mark_failed_with_errno(env, "could not open directory", result); + free(pathStr); + CloseHandle(handle); + return NULL; + } + + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)LocalAlloc(LPTR, sizeof(readdir_fast_handle_t)); + if (readdirHandle == NULL) { + mark_failed_with_code(env, "Out of native memory", ERROR_OUTOFMEMORY, NULL, result); + CloseHandle(handle); + free(pathStr); + return NULL; + } + readdirHandle->handle = handle; + readdirHandle->pathStr = pathStr; + readdirHandle->volumeSerialNumber = fileInfo.dwVolumeSerialNumber; + return (jlong)readdirHandle; +#endif +} + +// +// Releases all native resources associated with |handle|, a pointer to |fast_readdir_handle|. +// +JNIEXPORT void JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirClose(JNIEnv *env, jclass target, jlong handle) { +#ifdef WINDOWS_MIN + // Not supported +#else + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)handle; + CloseHandle(readdirHandle->handle); + free(readdirHandle->pathStr); + LocalFree(readdirHandle); +#endif +} + +// +// Returns the volume id of the directory opened by fastReaddirOpen +// +JNIEXPORT jint JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirGetVolumeId(JNIEnv *env, jclass target, jlong handle, jobject result) { +#ifdef WINDOWS_MIN + mark_failed_with_code(env, "Operation not supported", ERROR_CALL_NOT_IMPLEMENTED, NULL, result); + return 0; +#else + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)handle; + return readdirHandle->volumeSerialNumber; +#endif +} + +// +// Reads the next batch of entries from the directory. +// Returns JNI_TRUE on success and if there are more entries found +// Returns JNI_FALSE and sets an error to |result| if there is an error +// Returns JNI_FALSE if there are no more entries +// +JNIEXPORT jboolean JNICALL +Java_net_rubygrapefruit_platform_internal_jni_WindowsFileFunctions_fastReaddirNext(JNIEnv *env, jclass target, jlong handle, jobject buffer, jobject result) { +#ifdef WINDOWS_MIN + mark_failed_with_code(env, "Operation not supported", ERROR_CALL_NOT_IMPLEMENTED, NULL, result); + return JNI_FALSE; +#else + readdir_fast_handle_t* readdirHandle = (readdir_fast_handle_t*)handle; + + BYTE* entryBuffer = (BYTE*)env->GetDirectBufferAddress(buffer); + ULONG entryBufferSize = (ULONG)env->GetDirectBufferCapacity(buffer); + + NTSTATUS status = invokeNtQueryDirectoryFile(readdirHandle->handle, entryBuffer, entryBufferSize); + if (!NT_SUCCESS(status)) { + // Normal completion: no more files in directory + if (status == STATUS_NO_MORE_FILES) { + return JNI_FALSE; + } + + /* + * NtQueryDirectoryFile returns STATUS_INVALID_PARAMETER when + * asked to enumerate an invalid directory (ie it is a file + * instead of a directory). Verify that is the actual cause + * of the error. + */ + if (status == STATUS_INVALID_PARAMETER) { + DWORD attributes = GetFileAttributesW(readdirHandle->pathStr); + if ((attributes & FILE_ATTRIBUTE_DIRECTORY) == 0) { + status = STATUS_NOT_A_DIRECTORY; + } + } + mark_failed_with_ntstatus(env, "Error reading directory entries", status, result); + return JNI_FALSE; + } + + return JNI_TRUE; +#endif +} + /* * Console functions */ diff --git a/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java b/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java index 115fe750..f30048e1 100644 --- a/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java +++ b/src/main/java/net/rubygrapefruit/platform/file/FileInfo.java @@ -44,4 +44,12 @@ enum Type { * Returns the last modification time of this file, in ms since epoch. Returns 0 when this file does not exist. */ long getLastModifiedTime(); + + /** + * Returns an object that uniquely identifies the given file, or null if a file key is not available. + * + *

See BasicFileAttributes.fileKey() + * for a more in depth explanation.

+ */ + Object getKey(); } diff --git a/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java b/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java index fcbedfae..1c75b432 100644 --- a/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java +++ b/src/main/java/net/rubygrapefruit/platform/file/WindowsFileInfo.java @@ -25,4 +25,13 @@ */ @ThreadSafe public interface WindowsFileInfo extends FileInfo { + /** + * Returns the volume ID (serial number) of the file. + */ + int getVolumeId(); + + /** + * Returns the file ID of the file, unique within the volume identified by {@link #getVolumeId()}. + */ + long getFileId(); } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java index 5db7579b..f957b1ee 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/DefaultWindowsFiles.java @@ -24,9 +24,15 @@ import net.rubygrapefruit.platform.internal.jni.WindowsFileFunctions; import java.io.File; +import java.lang.ref.WeakReference; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.CharBuffer; import java.util.List; public class DefaultWindowsFiles extends AbstractFiles implements WindowsFiles { + private DirectoryLister directoryLister; + public WindowsFileInfo stat(File file) throws NativeException { return stat(file, false); } @@ -41,17 +47,207 @@ public WindowsFileInfo stat(File file, boolean linkTarget) throws NativeExceptio return stat; } + public List listDir(File dir) throws NativeException { + return listDir(dir, false); + } + public List listDir(File dir, boolean linkTarget) throws NativeException { - FunctionResult result = new FunctionResult(); - WindowsDirList dirList = new WindowsDirList(); - WindowsFileFunctions.readdir(dir.getPath(), linkTarget, dirList, result); - if (result.isFailed()) { - throw listDirFailure(dir, result); + if (directoryLister == null) { + directoryLister = WindowsFileFunctions.fastReaddirIsSupported() ? new FastLister() : new BackwardCompatibleLister(); } - return dirList.files; + return directoryLister.listDir(dir, linkTarget); } - public List listDir(File dir) throws NativeException { - return listDir(dir, false); + private interface DirectoryLister { + List listDir(File dir, boolean linkTarget) throws NativeException; + } + + private class BackwardCompatibleLister implements DirectoryLister { + public List listDir(File dir, boolean linkTarget) throws NativeException { + FunctionResult result = new FunctionResult(); + WindowsDirList dirList = new WindowsDirList(); + WindowsFileFunctions.readdir(dir.getPath(), linkTarget, dirList, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + return dirList.files; + } + } + + private class FastLister implements DirectoryLister { + private static final int FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400; + private static final int FILE_ATTRIBUTE_DIRECTORY = 0x00000010; + private static final int IO_REPARSE_TAG_SYMLINK = 0xA000000C; + + private static final int SIZEOF_WCHAR = 2; + private static final int OFFSETOF_NEXT_ENTRY_OFFSET = 0; + private static final int OFFSETOF_LAST_WRITE_TIME = 24; + private static final int OFFSETOF_END_OF_FILE = 40; + private static final int OFFSETOF_FILE_ATTRIBUTES = 56; + private static final int OFFSETOF_FILENAME_LENGTH = 60; + private static final int OFFSETOF_EA_SIZE = 64; + private static final int OFFSETOF_FILE_ID = 72; + private static final int OFFSETOF_FILENAME = 80; + + /** + * We use a thread local context to store a weak reference to a {@link NtQueryDirectoryFileContext} + * instance so that we can avoid an extra direct {@link ByteBuffer} allocation at every + * invocation of {@link #listDir(File, boolean)}. + *

This has shown to improve performance by a few percents in stress test benchmarks, while at the + * same time limiting memory usage increase to a minimum (about 8KB per thread using this API)

+ *

This is safe because {@link #listDir(File, boolean)} invocations are self-contained, i.e. don't + * call back into external code.

+ */ + private final ThreadLocal> threadLocalContext = + new ThreadLocal>(); + + public List listDir(File dir, boolean linkTarget) throws NativeException { + FunctionResult result = new FunctionResult(); + WindowsDirList dirList = new WindowsDirList(); + String path = dir.getPath(); + + long handle = WindowsFileFunctions.fastReaddirOpen(path, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + int volumeId = WindowsFileFunctions.fastReaddirGetVolumeId(handle, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + try { + NtQueryDirectoryFileContext context = getNtQueryDirectoryFileContext(); + + boolean more = WindowsFileFunctions.fastReaddirNext(handle, context.buffer, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + if (more) { + int entryOffset = 0; + while (true) { + // Read entry from buffer + entryOffset = addFullDirEntry(context, dir, linkTarget, volumeId, entryOffset, dirList); + + // If we reached end of buffer, fetch next set of entries + if (entryOffset == 0) { + more = WindowsFileFunctions.fastReaddirNext(handle, context.buffer, result); + if (result.isFailed()) { + throw listDirFailure(dir, result); + } + if (!more) { + break; + } + } + } + } + } finally { + WindowsFileFunctions.fastReaddirClose(handle); + } + + return dirList.files; + } + + /** + * Parse the content of {@link NtQueryDirectoryFileContext#buffer} at offset {@code entryOffset} + * as a FILE_ID_FULL_DIR_INFORMATION + * native structure and adds the parsed entry into the {@link WindowsDirList} collection. + *

Returns the byte offset of the next entry in {@link NtQueryDirectoryFileContext#buffer} if there is one, + * or {@code 0} if there is no next entry.

+ */ + private int addFullDirEntry(NtQueryDirectoryFileContext context, File dir, boolean followLink, int volumeId, int entryOffset, WindowsDirList dirList) { + // typedef struct _FILE_ID_FULL_DIR_INFORMATION { + // ULONG NextEntryOffset; // offset = 0 + // ULONG FileIndex; // offset = 4 + // LARGE_INTEGER CreationTime; // offset = 8 + // LARGE_INTEGER LastAccessTime; // offset = 16 + // LARGE_INTEGER LastWriteTime; // offset = 24 + // LARGE_INTEGER ChangeTime; // offset = 32 + // LARGE_INTEGER EndOfFile; // offset = 40 + // LARGE_INTEGER AllocationSize; // offset = 48 + // ULONG FileAttributes; // offset = 56 + // ULONG FileNameLength; // offset = 60 + // ULONG EaSize; // offset = 64 + // LARGE_INTEGER FileId; // offset = 72 + // WCHAR FileName[1]; // offset = 80 + //} FILE_ID_FULL_DIR_INFORMATION, *PFILE_ID_FULL_DIR_INFORMATION; + int nextEntryOffset = context.buffer.getInt(entryOffset + OFFSETOF_NEXT_ENTRY_OFFSET); + + long fileSize = context.buffer.getLong(entryOffset + OFFSETOF_END_OF_FILE); + long lastModified = context.buffer.getLong(entryOffset + OFFSETOF_LAST_WRITE_TIME); + + // + // See https://docs.microsoft.com/en-us/windows/desktop/fileio/reparse-point-tags + // IO_REPARSE_TAG_SYMLINK (0xA000000C) + // + int fileAttributes = context.buffer.getInt(entryOffset + OFFSETOF_FILE_ATTRIBUTES); + int reparseTagData = context.buffer.getInt(entryOffset + OFFSETOF_EA_SIZE); + long fileId = context.buffer.getLong(entryOffset + OFFSETOF_FILE_ID); + + FileInfo.Type type = getFileType(fileAttributes, reparseTagData); + + int FileNameByteCount = context.buffer.getInt(entryOffset + OFFSETOF_FILENAME_LENGTH); + context.charBuffer.position((entryOffset + OFFSETOF_FILENAME) / SIZEOF_WCHAR); + context.charBuffer.get(context.fileNameArray, 0, FileNameByteCount / SIZEOF_WCHAR); + String fileName = new String(context.fileNameArray, 0, FileNameByteCount / SIZEOF_WCHAR); + + // Skip "." and ".." entries + if (!".".equals(fileName) && !"..".equals(fileName)) { + if (type == FileInfo.Type.Symlink && followLink) { + WindowsFileInfo targetInfo = stat(new File(dir, fileName), true); + dirList.addFile(fileName, targetInfo); + } else { + dirList.addFile(fileName, type, fileSize, WindowsFileTime.toJavaTime(lastModified), volumeId, fileId); + } + } + + return nextEntryOffset == 0 ? 0 : entryOffset + nextEntryOffset; + } + + private FileInfo.Type getFileType(int dwFileAttributes, int reparseTagData) { + if (((dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT) && (reparseTagData == IO_REPARSE_TAG_SYMLINK)) { + return FileInfo.Type.Symlink; + } else if ((dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY) { + return FileInfo.Type.Directory; + } else { + return FileInfo.Type.File; + } + } + + private NtQueryDirectoryFileContext getNtQueryDirectoryFileContext() { + WeakReference ref = threadLocalContext.get(); + NtQueryDirectoryFileContext result = (ref == null ? null : ref.get()); + if (result == null) { + result = new NtQueryDirectoryFileContext(); + ref = new WeakReference(result); + threadLocalContext.set(ref); + } + return result; + } + + private class NtQueryDirectoryFileContext { + /** + * The {@code direct} {@link ByteBuffer} used to share memory between C++ and Java code. + */ + final ByteBuffer buffer; + /** + * A {@link CharBuffer} view of the {@link #buffer} above used to retrieve UTF-16 filename strings. + */ + final CharBuffer charBuffer; + /** + * A {@link java.lang.Character} array used to copy characters from the {@link #charBuffer} above + * before converting into a {@link java.lang.String} instance. + */ + final char[] fileNameArray; + + NtQueryDirectoryFileContext() { + // Note: 8KB is enough to store ~100 FILE_ID_FULL_DIR_INFORMATION entries, counting 80 bytes + // as "fixed" data plus 10 UTF-16 characters for the file name. + buffer = ByteBuffer.allocateDirect(8192); + // Win32/x86 is little endian + buffer.order(ByteOrder.LITTLE_ENDIAN); + charBuffer = buffer.asCharBuffer(); + // Note: Win32 file names are limited to 256 characters + fileNameArray = new char[256]; + } + } } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/DirList.java b/src/main/java/net/rubygrapefruit/platform/internal/DirList.java index f36e5509..5afbc292 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/DirList.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/DirList.java @@ -28,21 +28,37 @@ public class DirList { // Called from native code @SuppressWarnings("UnusedDeclaration") public void addFile(String name, int type, long size, long lastModified) { - DefaultDirEntry fileStat = new DefaultDirEntry(name, FileInfo.Type.values()[type], size, lastModified); + addFile(name, type, size, lastModified, 0 ,0); + } + + // Called from native code + @SuppressWarnings("UnusedDeclaration") + public void addFile(String name, int type, long size, long lastModified, int volumeId, long fileId) { + addFile(name, FileInfo.Type.values()[type], size, lastModified, volumeId, fileId); + } + + void addFile(String name, FileInfo.Type type, long size, long lastModified, int volumeId, long fileId) { + DefaultDirEntry fileStat = new DefaultDirEntry(name, type, size, lastModified, volumeId, fileId); files.add(fileStat); } - private static class DefaultDirEntry implements DirEntry { + protected static class DefaultDirEntry implements DirEntry { private final String name; private final Type type; private final long size; private final long lastModified; + private final int volumeId; + private final long fileId; + // Lazily initialized to avoid extra allocation if not needed + private volatile FileKey key; - DefaultDirEntry(String name, Type type, long size, long lastModified) { + DefaultDirEntry(String name, Type type, long size, long lastModified, int volumeId, long fileId) { this.name = name; this.type = type; this.size = size; this.lastModified = lastModified; + this.volumeId = volumeId; + this.fileId = fileId; } @Override @@ -65,5 +81,20 @@ public long getLastModifiedTime() { public long getSize() { return size; } + + public Object getKey() { + if (volumeId == 0 && fileId == 0) { + return null; + } + if (key == null) { + synchronized (this) { + if (key == null) { + key = new FileKey(volumeId, fileId); + } + } + } + return key; + } + } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/FileKey.java b/src/main/java/net/rubygrapefruit/platform/internal/FileKey.java new file mode 100644 index 00000000..aa9a8b0c --- /dev/null +++ b/src/main/java/net/rubygrapefruit/platform/internal/FileKey.java @@ -0,0 +1,40 @@ +package net.rubygrapefruit.platform.internal; + +final class FileKey { + private final int volumeId; + private final long fileId; + + FileKey(int volumeId, long fileId) { + this.volumeId = volumeId; + this.fileId = fileId; + } + + @Override + public int hashCode() { + return (int)(volumeId * 31 + fileId); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) + return true; + if (!(obj instanceof FileKey)) + return false; + + FileKey other = (FileKey) obj; + return (this.volumeId == other.volumeId) && + (this.fileId == other.fileId); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("(volumeId=") + .append(Integer.toHexString(volumeId)) + .append(",fileId=") + .append(Long.toHexString(fileId)) + .append(')'); + return sb.toString(); + } + +} diff --git a/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java b/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java index 52ea3025..fd6384b1 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/FileStat.java @@ -27,12 +27,17 @@ public class FileStat implements PosixFileInfo { private long size; private long modificationTime; private long blockSize; + private int volumeId; + private long fileId; + // Lazily initialized to avoid extra allocation if not needed + private volatile FileKey key; public FileStat(String path) { this.path = path; } - public void details(int type, int mode, int uid, int gid, long size, long modificationTime, int blockSize) { + // Called from native code + public void details(int type, int mode, int uid, int gid, long size, long modificationTime, int blockSize, int volumeId, long fileId) { this.type = Type.values()[type]; this.mode = mode; this.uid = uid; @@ -40,6 +45,8 @@ public void details(int type, int mode, int uid, int gid, long size, long modifi this.size = size; this.modificationTime = modificationTime; this.blockSize = blockSize; + this.volumeId = volumeId; + this.fileId = fileId; } @Override @@ -74,4 +81,26 @@ public long getBlockSize() { public long getLastModifiedTime() { return modificationTime; } + + public Object getKey() { + if (volumeId == 0 && fileId == 0) { + return null; + } + if (key == null) { + synchronized (this) { + if (key == null) { + key = new FileKey(volumeId, fileId); + } + } + } + return key; + } + + public int getVolumeId() { + return volumeId; + } + + public long getFileId() { + return fileId; + } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java index 319fac34..2588373f 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsDirList.java @@ -16,11 +16,18 @@ package net.rubygrapefruit.platform.internal; +import net.rubygrapefruit.platform.file.FileInfo; +import net.rubygrapefruit.platform.file.WindowsFileInfo; + public class WindowsDirList extends DirList { // Called from native code @SuppressWarnings("UnusedDeclaration") @Override public void addFile(String name, int type, long size, long lastModified) { - super.addFile(name, type, size, WindowsFileTime.toJavaTime(lastModified)); + addFile(name, FileInfo.Type.values()[type], size, WindowsFileTime.toJavaTime(lastModified), 0, 0); + } + + void addFile(String name, WindowsFileInfo fileInfo) { + addFile(name, fileInfo.getType(), fileInfo.getSize(), fileInfo.getLastModifiedTime(), fileInfo.getVolumeId(), fileInfo.getFileId()); } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java index 5e85b99d..684bcf4f 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/WindowsFileStat.java @@ -23,15 +23,27 @@ public class WindowsFileStat implements WindowsFileInfo { private Type type; private long size; private long lastModified; + private int volumeId; + private long fileId; + // Lazily initialized to avoid extra allocation if not needed + private volatile FileKey key; public WindowsFileStat(String path) { this.path = path; } + // Called from native code public void details(int type, long size, long lastModifiedWinTime) { + details(type, size,lastModifiedWinTime, 0, 0); + } + + // Called from native code + public void details(int type, long size, long lastModifiedWinTime, int volumeId, long fileId) { this.type = Type.values()[type]; this.size = size; this.lastModified = WindowsFileTime.toJavaTime(lastModifiedWinTime); + this.volumeId = volumeId; + this.fileId = fileId; } @Override @@ -50,4 +62,26 @@ public long getSize() { public long getLastModifiedTime() { return lastModified; } + + public int getVolumeId() { + return volumeId; + } + + public long getFileId() { + return fileId; + } + + public Object getKey() { + if (volumeId == 0 && fileId == 0) { + return null; + } + if (key == null) { + synchronized (this) { + if (key == null) { + key = new FileKey(volumeId, fileId); + } + } + } + return key; + } } diff --git a/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java b/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java index a192338f..ea142ce5 100644 --- a/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java +++ b/src/main/java/net/rubygrapefruit/platform/internal/jni/WindowsFileFunctions.java @@ -20,8 +20,17 @@ import net.rubygrapefruit.platform.internal.FunctionResult; import net.rubygrapefruit.platform.internal.WindowsFileStat; +import java.nio.ByteBuffer; + +@SuppressWarnings("SpellCheckingInspection") public class WindowsFileFunctions { public static native void stat(String file, boolean followLink, WindowsFileStat stat, FunctionResult result); public static native void readdir(String path, boolean followLink, DirList dirList, FunctionResult result); + + public static native boolean fastReaddirIsSupported(); + public static native long fastReaddirOpen(String path, FunctionResult result); + public static native void fastReaddirClose(long handle); + public static native int fastReaddirGetVolumeId(long handle, FunctionResult result); + public static native boolean fastReaddirNext(long handle, ByteBuffer buffer, FunctionResult result); } diff --git a/src/shared/headers/generic.h b/src/shared/headers/generic.h index 4722ee5e..08ff5d32 100755 --- a/src/shared/headers/generic.h +++ b/src/shared/headers/generic.h @@ -110,6 +110,8 @@ typedef struct file_stat { jint fileType; jlong lastModified; jlong size; + jint volumeId; + jlong fileId; } file_stat_t; #ifdef __cplusplus diff --git a/src/shared/headers/ntifs_min.h b/src/shared/headers/ntifs_min.h new file mode 100644 index 00000000..9206c38e --- /dev/null +++ b/src/shared/headers/ntifs_min.h @@ -0,0 +1,149 @@ +#ifndef _NTIFS_MIN_ +#define _NTIFS_MIN_ + +extern "C" { + +/* +* Copy necessary structures and definitions out of the Windows DDK +* to enable calling NtQueryDirectoryFile() +*/ + +typedef _Return_type_success_(return >= 0) LONG NTSTATUS; +#define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0) + +typedef struct _UNICODE_STRING { + USHORT Length; + USHORT MaximumLength; +#ifdef MIDL_PASS + [size_is(MaximumLength / 2), length_is((Length) / 2)] USHORT * Buffer; +#else // MIDL_PASS + _Field_size_bytes_part_(MaximumLength, Length) PWCH Buffer; +#endif // MIDL_PASS +} UNICODE_STRING; +typedef UNICODE_STRING *PUNICODE_STRING; +typedef const UNICODE_STRING *PCUNICODE_STRING; + +typedef enum _FILE_INFORMATION_CLASS { + FileDirectoryInformation = 1, + FileFullDirectoryInformation, + FileBothDirectoryInformation, + FileBasicInformation, + FileStandardInformation, + FileInternalInformation, + FileEaInformation, + FileAccessInformation, + FileNameInformation, + FileRenameInformation, + FileLinkInformation, + FileNamesInformation, + FileDispositionInformation, + FilePositionInformation, + FileFullEaInformation, + FileModeInformation, + FileAlignmentInformation, + FileAllInformation, + FileAllocationInformation, + FileEndOfFileInformation, + FileAlternateNameInformation, + FileStreamInformation, + FilePipeInformation, + FilePipeLocalInformation, + FilePipeRemoteInformation, + FileMailslotQueryInformation, + FileMailslotSetInformation, + FileCompressionInformation, + FileObjectIdInformation, + FileCompletionInformation, + FileMoveClusterInformation, + FileQuotaInformation, + FileReparsePointInformation, + FileNetworkOpenInformation, + FileAttributeTagInformation, + FileTrackingInformation, + FileIdBothDirectoryInformation, + FileIdFullDirectoryInformation, + FileValidDataLengthInformation, + FileShortNameInformation, + FileIoCompletionNotificationInformation, + FileIoStatusBlockRangeInformation, + FileIoPriorityHintInformation, + FileSfioReserveInformation, + FileSfioVolumeInformation, + FileHardLinkInformation, + FileProcessIdsUsingFileInformation, + FileNormalizedNameInformation, + FileNetworkPhysicalNameInformation, + FileIdGlobalTxDirectoryInformation, + FileIsRemoteDeviceInformation, + FileAttributeCacheInformation, + FileNumaNodeInformation, + FileStandardLinkInformation, + FileRemoteProtocolInformation, + FileMaximumInformation +} FILE_INFORMATION_CLASS, *PFILE_INFORMATION_CLASS; + +typedef struct _FILE_ID_FULL_DIR_INFORMATION { + ULONG NextEntryOffset; + ULONG FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + ULONG FileAttributes; + ULONG FileNameLength; + ULONG EaSize; + LARGE_INTEGER FileId; + WCHAR FileName[1]; +} FILE_ID_FULL_DIR_INFORMATION, *PFILE_ID_FULL_DIR_INFORMATION; + +typedef struct _IO_STATUS_BLOCK { + union { + NTSTATUS Status; + PVOID Pointer; + } u; + ULONG_PTR Information; +} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK; + +typedef VOID +(NTAPI *PIO_APC_ROUTINE)( + IN PVOID ApcContext, + IN PIO_STATUS_BLOCK IoStatusBlock, + IN ULONG Reserved); + +NTSYSCALLAPI +NTSTATUS +NTAPI +NtQueryDirectoryFile( + _In_ HANDLE FileHandle, + _In_opt_ HANDLE Event, + _In_opt_ PIO_APC_ROUTINE ApcRoutine, + _In_opt_ PVOID ApcContext, + _Out_ PIO_STATUS_BLOCK IoStatusBlock, + _Out_writes_bytes_(Length) PVOID FileInformation, + _In_ ULONG Length, + _In_ FILE_INFORMATION_CLASS FileInformationClass, + _In_ BOOLEAN ReturnSingleEntry, + _In_opt_ PUNICODE_STRING FileName, + _In_ BOOLEAN RestartScan +); + +// +// This function might be needed for some of the internal Windows functions, +// defined in this header file. +// +_When_(Status < 0, _Out_range_(>, 0)) +_When_(Status >= 0, _Out_range_(==, 0)) +ULONG +NTAPI +RtlNtStatusToDosError ( + NTSTATUS Status +); + +#define STATUS_NO_MORE_FILES ((NTSTATUS)0x80000006L) +#define STATUS_NOT_A_DIRECTORY ((NTSTATUS)0xC0000103L) + +} // extern "C" + +#endif // _NTIFS_MIN_ \ No newline at end of file diff --git a/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy b/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy index c3232fae..ceb54e32 100755 --- a/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy +++ b/src/test/groovy/net/rubygrapefruit/platform/file/FilesTest.groovy @@ -41,6 +41,7 @@ abstract class FilesTest extends AbstractFilesTest { void assertIsFile(FileInfo stat, File file) { assert stat.type == FileInfo.Type.File assert stat.size == file.length() + assert stat.key != null def attributes = attributes(file) assertTimestampMatches(stat.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(stat.lastModifiedTime, file.lastModified()) @@ -50,6 +51,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.type == FileInfo.Type.File assert entry.name == name assert entry.size == file.length() + assert entry.key != null def attributes = attributes(file) assertTimestampMatches(entry.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(entry.lastModifiedTime, file.lastModified()) @@ -58,6 +60,7 @@ abstract class FilesTest extends AbstractFilesTest { void assertIsDirectory(FileInfo stat, File file) { assert stat.type == FileInfo.Type.Directory assert stat.size == 0 + assert stat.key != null def attributes = attributes(file) assertTimestampMatches(stat.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(stat.lastModifiedTime, file.lastModified()) @@ -67,6 +70,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.type == FileInfo.Type.Directory assert entry.name == file.name assert entry.size == 0 + assert entry.key != null def attributes = attributes(file) assertTimestampMatches(entry.lastModifiedTime, attributes.lastModifiedTime().toMillis()) assertTimestampMatches(entry.lastModifiedTime, file.lastModified()) @@ -75,6 +79,7 @@ abstract class FilesTest extends AbstractFilesTest { void assertIsSymlink(FileInfo stat, File file) { assert stat.type == FileInfo.Type.Symlink assert stat.size == 0 + assert stat.key != null def attributes = attributes(file) assertTimestampMatches(stat.lastModifiedTime, attributes.lastModifiedTime().toMillis()) // Can't check `file.lastModified()` as it follows symlinks @@ -84,6 +89,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.type == FileInfo.Type.Symlink assert entry.name == file.name assert entry.size == 0 + assert entry.key != null def attributes = attributes(file) assertTimestampMatches(entry.lastModifiedTime, attributes.lastModifiedTime().toMillis()) // Can't check `file.lastModified()` as it follows symlinks @@ -93,6 +99,7 @@ abstract class FilesTest extends AbstractFilesTest { assert stat.type == FileInfo.Type.Missing assert stat.size == 0 assert stat.lastModifiedTime == 0 + assert stat.key == null } void assertIsMissing(DirEntry entry, String name) { @@ -100,6 +107,7 @@ abstract class FilesTest extends AbstractFilesTest { assert entry.name == name assert entry.size == 0 assert entry.lastModifiedTime == 0 + assert entry.key == null } def "caches file instance"() { @@ -495,6 +503,7 @@ abstract class FilesTest extends AbstractFilesTest { def linkEntry = files[2] assertIsSymlink(linkEntry, childLink) + assert linkEntry.key != fileEntry.key def missingEntry = files[3] assertIsSymlink(missingEntry, childMissingLink) @@ -541,6 +550,7 @@ abstract class FilesTest extends AbstractFilesTest { def linkEntry = files[2] assertIsFile(linkEntry, childFile, childLink.name) + assert linkEntry.key == fileEntry.key def missingEntry = files[3] assertIsMissing(missingEntry, childMissingLink.name) diff --git a/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java b/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java index 5a7137ea..831d71a2 100755 --- a/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java +++ b/test-app/src/main/java/net/rubygrapefruit/platform/test/Main.java @@ -404,6 +404,9 @@ private static void ls(String path, boolean followLinks) { System.out.println(); System.out.println("* Name: " + entry.getName()); System.out.println("* Type: " + entry.getType()); + if (entry.getKey() != null) { + System.out.println("* Key: " + entry.getKey()); + } stat(new File(dir, entry.getName()), entry); } } @@ -424,6 +427,9 @@ private static void stat(String path, boolean linkTarget) { System.out.println(); System.out.println("* File: " + file); System.out.println("* Type: " + stat.getType()); + if (stat.getKey() != null) { + System.out.println("* Key: " + stat.getKey()); + } if (stat.getType() != FileInfo.Type.Missing) { if (stat instanceof PosixFileInfo) { stat(file, (PosixFileInfo) stat);