diff --git a/build.zig b/build.zig index eaa52e6..dd7ebad 100644 --- a/build.zig +++ b/build.zig @@ -29,7 +29,9 @@ pub fn build(b: *std.build.Builder) !void { }); facil_lib.linkLibrary(facil_dep.artifact("facil.io")); - facil_lib.install(); + // facil_lib.install(); + const unused_facil_install_step = b.addInstallArtifact(facil_lib); + _ = unused_facil_install_step; inline for ([_]struct { name: []const u8, @@ -73,10 +75,10 @@ pub fn build(b: *std.build.Builder) !void { }); example.linkLibrary(facil_dep.artifact("facil.io")); - example.addModule("zap", zap_module); - const example_run = example.run(); + // const example_run = example.run(); + const example_run = b.addRunArtifact(example); example_run_step.dependOn(&example_run.step); // install the artifact - depending on the "example" @@ -89,6 +91,7 @@ pub fn build(b: *std.build.Builder) !void { // // authenticating http client for internal testing + // (facil.io based, not used anymore) // var http_client_exe = b.addExecutable(.{ .name = "http_client", @@ -128,6 +131,20 @@ pub fn build(b: *std.build.Builder) !void { auth_tests.addModule("zap", zap_module); auth_tests.step.dependOn(&http_client_runner_build_step.step); + const run_auth_tests = b.addRunArtifact(auth_tests); + const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&auth_tests.step); + test_step.dependOn(&run_auth_tests.step); + + // pkghash + // + var pkghash_exe = b.addExecutable(.{ + .name = "pkghash", + .root_source_file = .{ .path = "./tools/pkghash.zig" }, + .target = target, + .optimize = optimize, + }); + var pkghash_step = b.step("pkghash", "Build pkghash"); + const pkghash_build_step = b.addInstallArtifact(pkghash_exe); + pkghash_step.dependOn(&pkghash_build_step.step); } diff --git a/build.zig.zon b/build.zig.zon index 63d9591..3ff576b 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -4,8 +4,16 @@ .dependencies = .{ .@"facil.io" = .{ - .url = "https://github.com/zigzap/facil.io/archive/64a3fb6d66225d3ff6194e84623eb9d48bc3b6fb.tar.gz", - .hash = "1220da26a9450eb75ecdb93b5dd3dabfea53053734cb68c748f0426d445179bc7c92", + // temp workaround until zig's fetch is fixed, supporting GH's redirects + .url = "http://localhost:8000/zap-0.0.7.tar.gz", + + // this is how it should be: + //.url = "https://github.com/zigzap/facil.io/archive/refs/tags/zap-0.0.7.tar.gz", + .hash = "1220d03e0579bbb726efb8224ea289b26227bc421158b45c1b16a60b31bfa400ab33", + // our tool says: + //.hash = "1220bbc4738c846f3253ae98e1848514f2ed1f02ecc1c7a62076b6508f449e9b5f66", + + } } } diff --git a/flake.lock b/flake.lock index bd60b3b..82e979d 100644 --- a/flake.lock +++ b/flake.lock @@ -105,11 +105,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1682110011, - "narHash": "sha256-J1ArhCRJov3Ycflq7QcmpOzeqqYj39AjlcH77cUx/pQ=", + "lastModified": 1682121798, + "narHash": "sha256-jfQALrsRCIKFcS9JB5lpsqhifRXHSaslrtTENY+OnCY=", "owner": "nixos", "repo": "nixpkgs", - "rev": "624f65a2b164bc9fe47324606940ffe773196813", + "rev": "c3895680d29301a78dd19be91f9c1728c1c28df5", "type": "github" }, "original": { @@ -166,11 +166,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1682078883, - "narHash": "sha256-+SEv1PYV/Y84JpwjBw2+SuZQo+YZ/M6n+bMnLTSkAEo=", + "lastModified": 1682122954, + "narHash": "sha256-CliO/Z30NEQyqghbSWHknU7Fj2MHYLMN9bz88B5JBWo=", "owner": "mitchellh", "repo": "zig-overlay", - "rev": "457d4283ced353eb9be1a1a78a35a37994bf1512", + "rev": "803cc45ba8cbd8ce17bc1c8184c2cd3e37be3171", "type": "github" }, "original": { diff --git a/src/test_auth.zig b/src/test_auth.zig index 196a50e..a2cd9b1 100644 --- a/src/test_auth.zig +++ b/src/test_auth.zig @@ -122,66 +122,44 @@ fn endpoint_http_unauthorized(e: *Endpoints.SimpleEndpoint, r: zap.SimpleRequest // // http client code for in-process sending of http request // -fn setHeader(h: [*c]fio.http_s, name: []const u8, value: []const u8) !void { - const hname: fio.fio_str_info_s = .{ - .data = util.toCharPtr(name), - .len = name.len, - .capa = name.len, - }; - - const vname: fio.fio_str_info_s = .{ - .data = util.toCharPtr(value), - .len = value.len, - .capa = value.len, - }; - const ret = fio.http_set_header2(h, hname, vname); - - if (ret == 0) return; - return zap.HttpError.HttpSetHeader; -} -fn sendRequest() void { - const ret = zap.http_connect("http://127.0.0.1:3000/test", null, .{ - .on_response = on_response, - .on_request = null, - .on_upgrade = null, - .on_finish = null, - .udata = null, - .public_folder = null, - .public_folder_length = 0, - .max_header_size = 32 * 1024, - .max_body_size = 500 * 1024, - .max_clients = 1, - .tls = null, - .reserved1 = 0, - .reserved2 = 0, - .reserved3 = 0, - .ws_max_msg_size = 0, - .timeout = 5, - .ws_timeout = 0, - .log = 0, - .is_client = 1, - }); - // _ = ret; - std.debug.print("\nret = {d}\n", .{ret}); - zap.fio_start(.{ .threads = 1, .workers = 1 }); -} +const ClientAuthReqHeaderFields = struct { + auth: Authenticators.AuthScheme, + token: []const u8, +}; -fn on_response(r: [*c]fio.http_s) callconv(.C) void { - // the first time around, we need to complete the request. E.g. set headers. - if (r.*.status_str == zap.FIOBJ_INVALID) { - setHeader(r, "Authorization", "Bearer ABCDEFG") catch return; - zap.http_finish(r); - return; - } - const response = zap.http_req2str(r); - if (zap.fio2str(response)) |body| { - std.debug.print("{s}\n", .{body}); - } else { - std.debug.print("Oops\n", .{}); +fn makeRequest(a: std.mem.Allocator, url: []const u8, auth: ?ClientAuthReqHeaderFields) !void { + const uri = try std.Uri.parse(url); + + var h = std.http.Headers{ .allocator = a }; + defer h.deinit(); + + if (auth) |auth_fields| { + const authstring = try std.fmt.allocPrint(a, "{s}{s}", .{ auth_fields.auth.str(), auth_fields.token }); + defer a.free(authstring); + try h.append(auth_fields.auth.headerFieldStrHeader(), authstring); } + + var http_client: std.http.Client = .{ .allocator = a }; + defer http_client.deinit(); + + var req = try http_client.request(.GET, uri, h, .{}); + defer req.deinit(); + + try req.start(); + try req.do(); + // var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); + // var buffer: [1024]u8 = undefined; + // we know we won't receive a lot + // const len = try br.reader().readAll(&buffer); + // std.debug.print("RESPONSE:\n{s}\n", .{buffer[0..len]}); zap.fio_stop(); } + +fn makeRequestThread(a: std.mem.Allocator, url: []const u8, auth: ?ClientAuthReqHeaderFields) !std.Thread { + return try std.Thread.spawn(.{}, makeRequest, .{ a, url, auth }); +} + // // end of http client code // @@ -190,32 +168,6 @@ test "BearerAuthSingle authenticateRequest OK" { const a = std.testing.allocator; const token = "ABCDEFG"; - // - // Unfortunately, spawning a child process confuses facilio: - // - // 1. attempt: spawn curl process before we start facilio threads - // this doesn't work: facilio doesn't start up if we spawn a child process - // var p = std.ChildProcess.init(&.{ - // "bash", - // "-c", - // "sleep 10; curl -H \"Authorization: Bearer\"" ++ token ++ " http://localhost:3000/test -v", - // }, a); - // try p.spawn(); - - // 2. attempt: - // our custom client doesn't work either - // var p = std.ChildProcess.init(&.{ - // "bash", - // "-c", - // "sleep 3; ./zig-out/bin/http_client &", - // }, a); - // try p.spawn(); - // std.debug.print("done spawning\n", .{}); - - // 3. attempt: sending the request in-process - // this doesn't work either because facilio wants to be either server or client, gets confused, at least when we're doing it this way - // sendRequest(); - // setup listener var listener = zap.SimpleEndpointListener.init( a, @@ -248,10 +200,13 @@ test "BearerAuthSingle authenticateRequest OK" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("\n\n*******************************************\n", .{}); - std.debug.print("\n\nPlease run the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client_runner\n", .{}); - std.debug.print("\n\n*******************************************\n", .{}); + // std.debug.print("\n\n*******************************************\n", .{}); + // std.debug.print("\n\nPlease run the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client_runner\n", .{}); + // std.debug.print("\n\n*******************************************\n", .{}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = token }); + defer thread.join(); // start worker threads zap.start(.{ @@ -304,8 +259,11 @@ test "BearerAuthSingle authenticateRequest test-unauthorized" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("Waiting for the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Bearer invalid\r", .{}); + // std.debug.print("Waiting for the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Bearer invalid\r", .{}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = "invalid" }); + defer thread.join(); // start worker threads zap.start(.{ @@ -352,8 +310,11 @@ test "BearerAuthMulti authenticateRequest OK" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("Waiting for the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer invalid\r", .{}); + // std.debug.print("Waiting for the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer " ++ token ++ "\r", .{}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = token }); + defer thread.join(); // start worker threads zap.start(.{ @@ -400,8 +361,11 @@ test "BearerAuthMulti authenticateRequest test-unauthorized" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("Waiting for the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer invalid\r", .{}); + // std.debug.print("Waiting for the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer invalid\r", .{}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = "invalid" }); + defer thread.join(); // start worker threads zap.start(.{ @@ -453,8 +417,11 @@ test "BasicAuth Token68 authenticateRequest" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("Waiting for the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ token ++ "\r", .{}); + // std.debug.print("Waiting for the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ token ++ "\r", .{}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = token }); + defer thread.join(); // start worker threads zap.start(.{ @@ -506,8 +473,11 @@ test "BasicAuth Token68 authenticateRequest test-unauthorized" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("Waiting for the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ "invalid\r", .{}); + // std.debug.print("Waiting for the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ "invalid\r", .{}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = "invalid" }); + defer thread.join(); // start worker threads zap.start(.{ @@ -569,8 +539,11 @@ test "BasicAuth UserPass authenticateRequest" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("Waiting for the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic {s}\r", .{encoded}); + // std.debug.print("Waiting for the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic {s}\r", .{encoded}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = encoded }); + defer thread.join(); // start worker threads zap.start(.{ @@ -619,6 +592,7 @@ test "BasicAuth UserPass authenticateRequest test-unauthorized" { var encoder = std.base64.url_safe.Encoder; var buffer: [256]u8 = undefined; const encoded = encoder.encode(&buffer, token); + _ = encoded; // create authenticator const Authenticator = Authenticators.BasicAuth(Map, .UserPass); @@ -632,8 +606,11 @@ test "BasicAuth UserPass authenticateRequest test-unauthorized" { try listener.addEndpoint(auth_ep.getEndpoint()); listener.listen() catch {}; - std.debug.print("Waiting for the following:\n", .{}); - std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic {s}-invalid\r", .{encoded}); + // std.debug.print("Waiting for the following:\n", .{}); + // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic invalid\r", .{}); + + const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = "invalid" }); + defer thread.join(); // start worker threads zap.start(.{ diff --git a/tools/Manifest.zig b/tools/Manifest.zig new file mode 100644 index 0000000..6295ec8 --- /dev/null +++ b/tools/Manifest.zig @@ -0,0 +1,500 @@ +// borrowed from the zig sourcebase https://github.com/ziglang/zig +pub const basename = "build.zig.zon"; +pub const Hash = std.crypto.hash.sha2.Sha256; + +pub const Dependency = struct { + url: []const u8, + url_tok: Ast.TokenIndex, + hash: ?[]const u8, + hash_tok: Ast.TokenIndex, +}; + +pub const ErrorMessage = struct { + msg: []const u8, + tok: Ast.TokenIndex, + off: u32, +}; + +pub const MultihashFunction = enum(u16) { + identity = 0x00, + sha1 = 0x11, + @"sha2-256" = 0x12, + @"sha2-512" = 0x13, + @"sha3-512" = 0x14, + @"sha3-384" = 0x15, + @"sha3-256" = 0x16, + @"sha3-224" = 0x17, + @"sha2-384" = 0x20, + @"sha2-256-trunc254-padded" = 0x1012, + @"sha2-224" = 0x1013, + @"sha2-512-224" = 0x1014, + @"sha2-512-256" = 0x1015, + @"blake2b-256" = 0xb220, + _, +}; + +pub const multihash_function: MultihashFunction = switch (Hash) { + std.crypto.hash.sha2.Sha256 => .@"sha2-256", + else => @compileError("unreachable"), +}; +comptime { + // We avoid unnecessary uleb128 code in hexDigest by asserting here the + // values are small enough to be contained in the one-byte encoding. + assert(@enumToInt(multihash_function) < 127); + assert(Hash.digest_length < 127); +} +pub const multihash_len = 1 + 1 + Hash.digest_length; + +name: []const u8, +version: std.SemanticVersion, +dependencies: std.StringArrayHashMapUnmanaged(Dependency), + +errors: []ErrorMessage, +arena_state: std.heap.ArenaAllocator.State, + +pub const Error = Allocator.Error; + +pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { + const node_tags = ast.nodes.items(.tag); + const node_datas = ast.nodes.items(.data); + assert(node_tags[0] == .root); + const main_node_index = node_datas[0].lhs; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + errdefer arena_instance.deinit(); + + var p: Parse = .{ + .gpa = gpa, + .ast = ast, + .arena = arena_instance.allocator(), + .errors = .{}, + + .name = undefined, + .version = undefined, + .dependencies = .{}, + .buf = .{}, + }; + defer p.buf.deinit(gpa); + defer p.errors.deinit(gpa); + defer p.dependencies.deinit(gpa); + + p.parseRoot(main_node_index) catch |err| switch (err) { + error.ParseFailure => assert(p.errors.items.len > 0), + else => |e| return e, + }; + + return .{ + .name = p.name, + .version = p.version, + .dependencies = try p.dependencies.clone(p.arena), + .errors = try p.arena.dupe(ErrorMessage, p.errors.items), + .arena_state = arena_instance.state, + }; +} + +pub fn deinit(man: *Manifest, gpa: Allocator) void { + man.arena_state.promote(gpa).deinit(); + man.* = undefined; +} + +const hex_charset = "0123456789abcdef"; + +pub fn hex64(x: u64) [16]u8 { + var result: [16]u8 = undefined; + var i: usize = 0; + while (i < 8) : (i += 1) { + const byte = @truncate(u8, x >> @intCast(u6, 8 * i)); + result[i * 2 + 0] = hex_charset[byte >> 4]; + result[i * 2 + 1] = hex_charset[byte & 15]; + } + return result; +} + +test hex64 { + const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; + try std.testing.expectEqualStrings("[00efcdab78563412]", s); +} + +pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { + var result: [multihash_len * 2]u8 = undefined; + + result[0] = hex_charset[@enumToInt(multihash_function) >> 4]; + result[1] = hex_charset[@enumToInt(multihash_function) & 15]; + + result[2] = hex_charset[Hash.digest_length >> 4]; + result[3] = hex_charset[Hash.digest_length & 15]; + + for (digest, 0..) |byte, i| { + result[4 + i * 2] = hex_charset[byte >> 4]; + result[5 + i * 2] = hex_charset[byte & 15]; + } + return result; +} + +const Parse = struct { + gpa: Allocator, + ast: std.zig.Ast, + arena: Allocator, + buf: std.ArrayListUnmanaged(u8), + errors: std.ArrayListUnmanaged(ErrorMessage), + + name: []const u8, + version: std.SemanticVersion, + dependencies: std.StringArrayHashMapUnmanaged(Dependency), + + const InnerError = error{ ParseFailure, OutOfMemory }; + + fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const main_token = main_tokens[node]; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + return fail(p, main_token, "expected top level expression to be a struct", .{}); + }; + + var have_name = false; + var have_version = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "dependencies")) { + try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "name")) { + p.name = try parseString(p, field_init); + have_name = true; + } else if (mem.eql(u8, field_name, "version")) { + const version_text = try parseString(p, field_init); + p.version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v undefined; + }; + have_version = true; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_name) { + try appendError(p, main_token, "missing top-level 'name' field", .{}); + } + + if (!have_version) { + try appendError(p, main_token, "missing top-level 'version' field", .{}); + } + } + + fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependencies expression to be a struct", .{}); + }; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try identifierTokenString(p, name_token); + const dep = try parseDependency(p, field_init); + try p.dependencies.put(p.gpa, dep_name, dep); + } + } + + fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependency expression to be a struct", .{}); + }; + + var dep: Dependency = .{ + .url = undefined, + .url_tok = undefined, + .hash = null, + .hash_tok = undefined, + }; + var have_url = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "url")) { + dep.url = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.url_tok = main_tokens[field_init]; + have_url = true; + } else if (mem.eql(u8, field_name, "hash")) { + dep.hash = parseHash(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.hash_tok = main_tokens[field_init]; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_url) { + try appendError(p, main_tokens[node], "dependency is missing 'url' field", .{}); + } + + return dep; + } + + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const node_tags = ast.nodes.items(.tag); + const main_tokens = ast.nodes.items(.main_token); + if (node_tags[node] != .string_literal) { + return fail(p, main_tokens[node], "expected string literal", .{}); + } + const str_lit_token = main_tokens[node]; + const token_bytes = ast.tokenSlice(str_lit_token); + p.buf.clearRetainingCapacity(); + try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const tok = main_tokens[node]; + const h = try parseString(p, node); + + if (h.len >= 2) { + const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { + return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{ + @errorName(err), + }); + }; + if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) { + return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{}); + } + } + + const hex_multihash_len = 2 * Manifest.multihash_len; + if (h.len != hex_multihash_len) { + return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ + hex_multihash_len, h.len, + }); + } + + return h; + } + + /// TODO: try to DRY this with AstGen.identifierTokenString + fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { + const ast = p.ast; + const token_tags = ast.tokens.items(.tag); + assert(token_tags[token] == .identifier); + const ident_name = ast.tokenSlice(token); + if (!mem.startsWith(u8, ident_name, "@")) { + return ident_name; + } + p.buf.clearRetainingCapacity(); + try parseStrLit(p, token, &p.buf, ident_name, 1); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + /// TODO: try to DRY this with AstGen.parseStrLit + fn parseStrLit( + p: *Parse, + token: Ast.TokenIndex, + buf: *std.ArrayListUnmanaged(u8), + bytes: []const u8, + offset: u32, + ) InnerError!void { + const raw_string = bytes[offset..]; + var buf_managed = buf.toManaged(p.gpa); + const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); + buf.* = buf_managed.moveToUnmanaged(); + switch (try result) { + .success => {}, + .failure => |err| try p.appendStrLitError(err, token, bytes, offset), + } + } + + /// TODO: try to DRY this with AstGen.failWithStrLitError + fn appendStrLitError( + p: *Parse, + err: std.zig.string_literal.Error, + token: Ast.TokenIndex, + bytes: []const u8, + offset: u32, + ) Allocator.Error!void { + const raw_string = bytes[offset..]; + switch (err) { + .invalid_escape_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "invalid escape character: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .expected_hex_digit => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected hex digit, found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_unicode_escape_sequence => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "empty unicode escape sequence", + .{}, + ); + }, + .expected_hex_digit_or_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected hex digit or '}}', found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .invalid_unicode_codepoint => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "unicode escape does not correspond to a valid codepoint", + .{}, + ); + }, + .expected_lbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected '{{', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected '}}', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_single_quote => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected single quote ('), found '{c}", + .{raw_string[bad_index]}, + ); + }, + .invalid_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "invalid byte in string or character literal: '{c}'", + .{raw_string[bad_index]}, + ); + }, + } + } + + fn fail( + p: *Parse, + tok: Ast.TokenIndex, + comptime fmt: []const u8, + args: anytype, + ) InnerError { + try appendError(p, tok, fmt, args); + return error.ParseFailure; + } + + fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return appendErrorOff(p, tok, 0, fmt, args); + } + + fn appendErrorOff( + p: *Parse, + tok: Ast.TokenIndex, + byte_offset: u32, + comptime fmt: []const u8, + args: anytype, + ) Allocator.Error!void { + try p.errors.append(p.gpa, .{ + .msg = try std.fmt.allocPrint(p.arena, fmt, args), + .tok = tok, + .off = byte_offset, + }); + } +}; + +const Manifest = @This(); +const std = @import("std"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Ast = std.zig.Ast; +const testing = std.testing; + +test "basic" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + \\ }, + \\ }, + \\} + ; + + var ast = try std.zig.Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast); + defer manifest.deinit(gpa); + + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 3, + .minor = 2, + .patch = 1, + }), manifest.version); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].url, + ); + try testing.expectEqualStrings( + "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + manifest.dependencies.values()[0].hash orelse return error.TestFailed, + ); +} diff --git a/tools/pkghash.zig b/tools/pkghash.zig new file mode 100644 index 0000000..f1617ea --- /dev/null +++ b/tools/pkghash.zig @@ -0,0 +1,233 @@ +const std = @import("std"); +const builtin = std.builtin; +const assert = std.debug.assert; +const io = std.io; +const fs = std.fs; +const mem = std.mem; +const process = std.process; +const Allocator = mem.Allocator; +const ThreadPool = std.Thread.Pool; +const WaitGroup = std.Thread.WaitGroup; + +const Manifest = @import("Manifest.zig"); + +var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){}; + +pub fn fatal(comptime format: []const u8, args: anytype) noreturn { + std.log.err(format, args); + process.exit(1); +} + +pub fn main() !void { + const gpa = general_purpose_allocator.allocator(); + defer _ = general_purpose_allocator.deinit(); + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + const args = try process.argsAlloc(arena); + try cmdPkg(gpa, arena, args); +} +pub const usage_pkg = + \\Usage: zig pkg [command] [options] + \\ + \\Options: + \\ -h --help Print this help and exit. + \\ + \\Sub-options for [hash]: + \\ --allow-directory : calc hash even if no build.zig is present + \\ +; + +pub fn cmdPkg(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + _ = arena; + if (args.len == 0) fatal("Expected at least one argument.\n", .{}); + const command_arg = args[0]; + + if (mem.eql(u8, command_arg, "-h") or mem.eql(u8, command_arg, "--help")) { + const stdout = io.getStdOut().writer(); + try stdout.writeAll(usage_pkg); + return; + } + + const cwd = std.fs.cwd(); + + dir_test: { + if (args.len > 1 and mem.eql(u8, args[1], "--allow-directory")) break :dir_test; + try if (cwd.access("build.zig", .{})) |_| break :dir_test else |err| switch (err) { + error.FileNotFound => {}, + else => |e| e, + }; + try if (cwd.access("build.zig.zon", .{})) |_| break :dir_test else |err| switch (err) { + error.FileNotFound => {}, + else => |e| e, + }; + break :dir_test fatal("Could not find either build.zig or build.zig.zon in this directory.\n Use --allow-directory to override this check.\n", .{}); + } + + const hash = blk: { + const cwd_absolute_path = try cwd.realpathAlloc(gpa, "."); + defer gpa.free(cwd_absolute_path); + + // computePackageHash will close the directory after completion + // std.debug.print("abspath: {s}\n", .{cwd_absolute_path}); + var cwd_copy = try fs.openIterableDirAbsolute(cwd_absolute_path, .{}); + errdefer cwd_copy.dir.close(); + + var thread_pool: ThreadPool = undefined; + try thread_pool.init(.{ .allocator = gpa }); + defer thread_pool.deinit(); + + // workaround for missing inclusion/exclusion support -> #14311. + const excluded_directories: []const []const u8 = &.{ + "zig-out", + "zig-cache", + ".git", + }; + break :blk try computePackageHashExcludingDirectories( + &thread_pool, + .{ .dir = cwd_copy.dir }, + excluded_directories, + ); + }; + + const std_out = std.io.getStdOut(); + const digest = Manifest.hexDigest(hash); + try std_out.writeAll(digest[0..]); + try std_out.writeAll("\n"); +} + +/// Make a file system path identical independently of operating system path inconsistencies. +/// This converts backslashes into forward slashes. +fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { + const canonical_sep = '/'; + + if (fs.path.sep == canonical_sep) + return fs_path; + + const normalized = try arena.dupe(u8, fs_path); + for (normalized) |*byte| { + switch (byte.*) { + fs.path.sep => byte.* = canonical_sep, + else => continue, + } + } + return normalized; +} + +const HashedFile = struct { + fs_path: []const u8, + normalized_path: []const u8, + hash: [Manifest.Hash.digest_length]u8, + failure: Error!void, + + const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError; + + fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { + _ = context; + return mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); + } +}; + +fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { + defer wg.finish(); + hashed_file.failure = hashFileFallible(dir, hashed_file); +} + +fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { + var buf: [8000]u8 = undefined; + var file = try dir.openFile(hashed_file.fs_path, .{}); + defer file.close(); + var hasher = Manifest.Hash.init(.{}); + hasher.update(hashed_file.normalized_path); + hasher.update(&.{ 0, @boolToInt(try isExecutable(file)) }); + while (true) { + const bytes_read = try file.read(&buf); + if (bytes_read == 0) break; + hasher.update(buf[0..bytes_read]); + } + hasher.final(&hashed_file.hash); +} + +fn isExecutable(file: fs.File) !bool { + _ = file; + // hack: in order to mimic current zig's tar extraction, we set everything to + // NOT EXECUTABLE + // const stat = try file.stat(); + // return (stat.mode & std.os.S.IXUSR) != 0; + return false; +} + +pub fn computePackageHashExcludingDirectories( + thread_pool: *ThreadPool, + pkg_dir: fs.IterableDir, + excluded_directories: []const []const u8, +) ![Manifest.Hash.digest_length]u8 { + const gpa = thread_pool.allocator; + + // We'll use an arena allocator for the path name strings since they all + // need to be in memory for sorting. + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + // Collect all files, recursively, then sort. + var all_files = std.ArrayList(*HashedFile).init(gpa); + defer all_files.deinit(); + + var walker = try pkg_dir.walk(gpa); + defer walker.deinit(); + + { + // The final hash will be a hash of each file hashed independently. This + // allows hashing in parallel. + var wait_group: WaitGroup = .{}; + defer wait_group.wait(); + + loop: while (try walker.next()) |entry| { + switch (entry.kind) { + .Directory => { + for (excluded_directories) |dir_name| { + if (mem.eql(u8, entry.basename, dir_name)) { + var item = walker.stack.pop(); + if (walker.stack.items.len != 0) { + item.iter.dir.close(); + } + continue :loop; + } + } + continue :loop; + }, + .File => {}, + else => return error.IllegalFileTypeInPackage, + } + const hashed_file = try arena.create(HashedFile); + const fs_path = try arena.dupe(u8, entry.path); + hashed_file.* = .{ + .fs_path = fs_path, + .normalized_path = try normalizePath(arena, fs_path), + .hash = undefined, // to be populated by the worker + .failure = undefined, // to be populated by the worker + }; + wait_group.start(); + try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); + + try all_files.append(hashed_file); + } + } + + std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan); + + var hasher = Manifest.Hash.init(.{}); + var any_failures = false; + for (all_files.items) |hashed_file| { + hashed_file.failure catch |err| { + any_failures = true; + std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) }); + }; + // std.debug.print("{s} : {s}\n", .{ hashed_file.normalized_path, Manifest.hexDigest(hashed_file.hash) }); + hasher.update(&hashed_file.hash); + } + if (any_failures) return error.PackageHashUnavailable; + return hasher.finalResult(); +} diff --git a/tools/server.py b/tools/server.py new file mode 100644 index 0000000..91f8b7c --- /dev/null +++ b/tools/server.py @@ -0,0 +1,19 @@ +import http.server as SimpleHTTPServer +import socketserver as SocketServer +import logging + +PORT = 8080 + +class GetHandler( + SimpleHTTPServer.SimpleHTTPRequestHandler + ): + + def do_GET(self): + logging.error(self.headers) + SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + + +Handler = GetHandler +httpd = SocketServer.TCPServer(("", PORT), Handler) + +httpd.serve_forever()