Skip to content

Amortize std.crypto.Certificate.Bundle rescanning in std.http.Client #25264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Amortize std.crypto.Certificate.Bundle rescanning in std.http.Client #25264

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1df9705
load a std.crypto.Certificate.Bundle from file or buffer
benburkert Sep 17, 2025
a564111
cleanup loading bundles from files & bytes, fix u64 to usize on 32bit
benburkert Sep 17, 2025
f962cf6
add a callback source to Certificate.Bundle for efficient rescanning
benburkert Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 149 additions & 26 deletions lib/std/crypto/Certificate/Bundle.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,58 @@
//! index from the DER-encoded subject name to the index of the containing
//! certificate within `bytes`.

source: Source,

/// The key is the contents slice of the subject.
map: std.HashMapUnmanaged(der.Element.Slice, u32, MapContext, std.hash_map.default_max_load_percentage) = .empty,
bytes: std.ArrayListUnmanaged(u8) = .empty,

pub const default: Bundle = .{
.source = .system,
};

/// No ca verification is performed, which prevents a trusted connection from
/// being established.
pub const no_verification = .{
.source = .{ .callback = &noVerification },
};

fn noVerification(_: *Bundle, _: Certificate.Parsed, _: i64) VerifyError!void {}

/// Verify that the server certificate is a valid self-signed certificate. This
/// provides no authorization guarantees, as anyone can create a self-signed
/// certificate.
pub const self_signed = .{
.source = .{ .callback = &selfSigned },
};

fn selfSigned(_: *Bundle, subject: Certificate.Parsed, now_sec: i64) VerifyError!void {
try subject.verify(subject, now_sec);
}

pub const Source = union(enum) {
system,
file: []const u8,
bytes: []const u8,
callback: *const fn (cb: *Bundle, subject: Certificate.Parsed, now_sec: i64) VerifyError!void,
};

pub fn init(allocator: Allocator, source: Source) !Bundle {
var bundle: Bundle = .{
.source = source,
};
try bundle.rescan(allocator);
return bundle;
}

pub const VerifyError = Certificate.Parsed.VerifyError || error{
CertificateIssuerNotFound,
};

pub fn verify(cb: Bundle, subject: Certificate.Parsed, now_sec: i64) VerifyError!void {
pub fn verify(cb: *Bundle, subject: Certificate.Parsed, now_sec: i64) VerifyError!void {
if (cb.source == .callback)
return cb.source.callback(cb, subject, now_sec);

const bytes_index = cb.find(subject.issuer()) orelse return error.CertificateIssuerNotFound;
const issuer_cert: Certificate = .{
.buffer = cb.bytes.items,
Expand All @@ -25,6 +68,14 @@ pub fn verify(cb: Bundle, subject: Certificate.Parsed, now_sec: i64) VerifyError
try subject.verify(issuer, now_sec);
}

pub const VerifyRescanError = VerifyError || RescanError;

pub fn verifyRescan(cb: *Bundle, allocator: Allocator, subject: Certificate.Parsed, now_sec: i64) VerifyRescanError!void {
_ = cb.find(subject.issuer()) orelse try cb.rescan(allocator);

return cb.verify(subject, now_sec);
}

/// The returned bytes become invalid after calling any of the rescan functions
/// or add functions.
pub fn find(cb: Bundle, subject_name: []const u8) ?u32 {
Expand Down Expand Up @@ -57,18 +108,23 @@ pub const RescanError = RescanLinuxError || RescanMacError || RescanWithPathErro
/// For operating systems that do not have standard CA installations to be
/// found, this function clears the set of certificates.
pub fn rescan(cb: *Bundle, gpa: Allocator) RescanError!void {
switch (builtin.os.tag) {
.linux => return rescanLinux(cb, gpa),
.macos => return rescanMac(cb, gpa),
.freebsd, .openbsd => return rescanWithPath(cb, gpa, "/etc/ssl/cert.pem"),
.netbsd => return rescanWithPath(cb, gpa, "/etc/openssl/certs/ca-certificates.crt"),
.dragonfly => return rescanWithPath(cb, gpa, "/usr/local/etc/ssl/cert.pem"),
.solaris, .illumos => return rescanWithPath(cb, gpa, "/etc/ssl/cacert.pem"),
.haiku => return rescanWithPath(cb, gpa, "/boot/system/data/ssl/CARootCertificates.pem"),
// https://github.com/SerenityOS/serenity/blob/222acc9d389bc6b490d4c39539761b043a4bfcb0/Ports/ca-certificates/package.sh#L19
.serenity => return rescanWithPath(cb, gpa, "/etc/ssl/certs/ca-certificates.crt"),
.windows => return rescanWindows(cb, gpa),
else => {},
switch (cb.source) {
.system => switch (builtin.os.tag) {
.linux => return rescanLinux(cb, gpa),
.macos => return rescanMac(cb, gpa),
.freebsd, .openbsd => return rescanWithPath(cb, gpa, "/etc/ssl/cert.pem"),
.netbsd => return rescanWithPath(cb, gpa, "/etc/openssl/certs/ca-certificates.crt"),
.dragonfly => return rescanWithPath(cb, gpa, "/usr/local/etc/ssl/cert.pem"),
.solaris, .illumos => return rescanWithPath(cb, gpa, "/etc/ssl/cacert.pem"),
.haiku => return rescanWithPath(cb, gpa, "/boot/system/data/ssl/CARootCertificates.pem"),
// https://github.com/SerenityOS/serenity/blob/222acc9d389bc6b490d4c39539761b043a4bfcb0/Ports/ca-certificates/package.sh#L19
.serenity => return rescanWithPath(cb, gpa, "/etc/ssl/certs/ca-certificates.crt"),
.windows => return rescanWindows(cb, gpa),
else => {},
},
.file => |path| return rescanWithPath(cb, gpa, path),
.bytes => |buffer| return rescanWithBytes(cb, gpa, buffer),
.callback => {},
}
}

Expand Down Expand Up @@ -155,6 +211,13 @@ fn rescanWindows(cb: *Bundle, gpa: Allocator) RescanWindowsError!void {
cb.bytes.shrinkAndFree(gpa, cb.bytes.items.len);
}

fn rescanWithBytes(cb: *Bundle, gpa: Allocator, buffer: []const u8) AddCertsFromBytesError!void {
cb.bytes.clearRetainingCapacity();
cb.map.clearRetainingCapacity();
try addCertsFromBytes(cb, gpa, buffer);
cb.bytes.shrinkAndFree(gpa, cb.bytes.items.len);
}

pub const AddCertsFromDirPathError = fs.File.OpenError || AddCertsFromDirError;

pub fn addCertsFromDirPath(
Expand Down Expand Up @@ -220,30 +283,45 @@ pub fn addCertsFromFilePath(
pub const AddCertsFromFileError = Allocator.Error ||
fs.File.GetSeekPosError ||
fs.File.ReadError ||
ParseCertError ||
std.base64.Error ||
error{ CertificateAuthorityBundleTooBig, MissingEndCertificateMarker };
AddCertsFromBytesError;

pub fn addCertsFromFile(cb: *Bundle, gpa: Allocator, file: fs.File) AddCertsFromFileError!void {
const size = try file.getEndPos();
const size = std.math.cast(usize, try file.getEndPos()) orelse
return error.CertificateAuthorityBundleTooBig;

// We borrow `bytes` as a temporary buffer for the base64-encoded data.
// This is possible by computing the decoded length and reserving the space
// for the decoded bytes first.
const decoded_size_upper_bound = size / 4 * 3;
// We borrow `bytes` as a temporary buffer for both the decoded certificate
// data and the encoded data from `file`. This is possible by placing the
// file data after the decoded data filled by addCertsFromBytes.
const decoded_size_upper_bound = try std.base64.standard.Decoder.calcSizeUpperBound(size);
const needed_capacity = std.math.cast(u32, decoded_size_upper_bound + size) orelse
return error.CertificateAuthorityBundleTooBig;
try cb.bytes.ensureUnusedCapacity(gpa, needed_capacity);

const end_reserved: u32 = @intCast(cb.bytes.items.len + decoded_size_upper_bound);
const buffer = cb.bytes.allocatedSlice()[end_reserved..];
const end_index = try file.readAll(buffer);
const encoded_bytes = buffer[0..end_index];

return cb.addCertsFromBytes(gpa, encoded_bytes);
}

pub const AddCertsFromBytesError = Allocator.Error ||
ParseCertError ||
std.base64.Error ||
error{ CertificateAuthorityBundleTooBig, MissingEndCertificateMarker };

pub fn addCertsFromBytes(cb: *Bundle, gpa: Allocator, encoded_bytes: []const u8) AddCertsFromBytesError!void {
const begin_marker = "-----BEGIN CERTIFICATE-----";
const end_marker = "-----END CERTIFICATE-----";

const now_sec = std.time.timestamp();

// We borrow `bytes` as a temporary buffer for the decoded certificate data.
const decoded_size_upper_bound = try std.base64.standard.Decoder.calcSizeUpperBound(encoded_bytes.len);
const needed_capacity = std.math.cast(u32, decoded_size_upper_bound) orelse
return error.CertificateAuthorityBundleTooBig;
try cb.bytes.ensureUnusedCapacity(gpa, needed_capacity);

var start_index: usize = 0;
while (mem.indexOfPos(u8, encoded_bytes, start_index, begin_marker)) |begin_marker_start| {
const cert_start = begin_marker_start + begin_marker.len;
Expand Down Expand Up @@ -318,11 +396,56 @@ const MapContext = struct {
}
};

test "scan for OS-provided certificates" {
if (builtin.os.tag == .wasi) return error.SkipZigTest;
test "load certificate bundle" {
// load from a file

{
var tmp_dir = std.testing.tmpDir(.{});
defer tmp_dir.cleanup();
var dir = tmp_dir.dir;

try dir.writeFile(.{ .sub_path = "cacert.pem", .data = cacert_pem });
const cacert_path = try dir.realpathAlloc(std.testing.allocator, "cacert.pem");
defer std.testing.allocator.free(cacert_path);

var bundle: Bundle = try .init(std.testing.allocator, .{
.file = cacert_path,
});
defer bundle.deinit(std.testing.allocator);
}

// load from a byte buffer

{
var bundle: Bundle = try .init(std.testing.allocator, .{
.bytes = cacert_pem,
});
defer bundle.deinit(std.testing.allocator);
}

// load from the system

var bundle: Bundle = .{};
defer bundle.deinit(std.testing.allocator);
{
if (builtin.os.tag == .wasi) return error.SkipZigTest;

try bundle.rescan(std.testing.allocator);
var bundle: Bundle = try .init(std.testing.allocator, .system);
defer bundle.deinit(std.testing.allocator);
}
}

// go run github.com/jsha/[email protected] -domains localhost
const cacert_pem =
\\-----BEGIN CERTIFICATE-----
\\MIIB/DCCAYKgAwIBAgIIYcxXcUcpGuYwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV
\\bWluaWNhIHJvb3QgY2EgNjFjYzU3MCAXDTI1MDkxNzAzNDAyNVoYDzIxMjUwOTE3
\\MDM0MDI1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA2MWNjNTcwdjAQBgcq
\\hkjOPQIBBgUrgQQAIgNiAARubFYDHoLNmM68GulcjVxxGxmqpNvosnDHpbBbU3wq
\\pzwYN5FXK2QdSy3MBHvNfyu2VZVYiNGyaIWz66vOh0f6dVPLXlo1ghRvMwnaP+qy
\\Xj8dWcedNoT2mxybVwxLOiKjgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW
\\MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud
\\DgQWBBSmzZRaHagE1f+5ADaaNlpYLmyV/jAfBgNVHSMEGDAWgBSmzZRaHagE1f+5
\\ADaaNlpYLmyV/jAKBggqhkjOPQQDAwNoADBlAjEA9kXs6mZXpu1MQz0GKv0aHEdo
\\swy3hE/7Y/UDy/bo71G3Qss1AjsS/flvfMNPIOecAjAx1wFTjdXS58CB02dTNXRv
\\BPPkWAiU7avvE1RsEQU2fvudhnoiVa8PDs0TJODFiR4=
\\-----END CERTIFICATE-----
;
35 changes: 7 additions & 28 deletions lib/std/crypto/tls/Client.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,17 +94,7 @@ pub const Options = struct {
explicit: []const u8,
},
/// How to verify the authenticity of server certificates.
ca: union(enum) {
/// No ca verification is performed, which prevents a trusted connection from
/// being established.
no_verification,
/// Verify that the server certificate is a valid self-signed certificate.
/// This provides no authorization guarantees, as anyone can create a
/// self-signed certificate.
self_signed,
/// Verify that the server certificate is authorized by a given ca bundle.
bundle: Certificate.Bundle,
},
ca_bundle: *Certificate.Bundle,
/// If non-null, ssl secrets are logged to this stream. Creating such a log file allows
/// other programs with access to that file to decrypt all traffic over this connection.
///
Expand Down Expand Up @@ -640,23 +630,12 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
try prev_cert.verify(subject, now_sec);
}

switch (options.ca) {
.no_verification => {
handshake_state = .trust_chain_established;
break :cert;
},
.self_signed => {
try subject.verify(subject, now_sec);
handshake_state = .trust_chain_established;
break :cert;
},
.bundle => |ca_bundle| if (ca_bundle.verify(subject, now_sec)) |_| {
handshake_state = .trust_chain_established;
break :cert;
} else |err| switch (err) {
error.CertificateIssuerNotFound => {},
else => |e| return e,
},
if (options.ca_bundle.verify(subject, now_sec)) |_| {
handshake_state = .trust_chain_established;
break :cert;
} else |err| switch (err) {
error.CertificateIssuerNotFound => {},
else => |e| return e,
}

prev_cert = subject;
Expand Down
36 changes: 19 additions & 17 deletions lib/std/http/Client.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const mem = std.mem;
const net = std.net;
const Uri = std.Uri;
const Allocator = mem.Allocator;
const Certificate = std.crypto.Certificate;
const assert = std.debug.assert;
const Writer = std.Io.Writer;
const Reader = std.Io.Reader;
Expand All @@ -23,7 +24,9 @@ pub const disable_tls = std.options.http_disable_tls;
/// Used for all client allocations. Must be thread-safe.
allocator: Allocator,

ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{},
ca_bundle: if (disable_tls) void else Certificate.Bundle = if (disable_tls) {} else .{
.source = .{ .callback = &caVerify },
},
ca_bundle_mutex: std.Thread.Mutex = .{},
/// Used both for the reader and writer buffers.
tls_buffer_size: if (disable_tls) u0 else usize = if (disable_tls) 0 else std.crypto.tls.Client.min_buffer_len,
Expand All @@ -32,10 +35,6 @@ tls_buffer_size: if (disable_tls) u0 else usize = if (disable_tls) 0 else std.cr
/// traffic over connections created with this `Client`.
ssl_key_log: ?*std.crypto.tls.Client.SslKeyLog = null,

/// When this is `true`, the next time this client performs an HTTPS request,
/// it will first rescan the system for root certificates.
next_https_rescan_certs: bool = true,

/// The pool of connections that can be reused (and currently in use).
connection_pool: ConnectionPool = .{},
/// Each `Connection` allocates this amount for the reader buffer.
Expand Down Expand Up @@ -325,13 +324,12 @@ pub const Connection = struct {
.closing = false,
.protocol = .tls,
},
// TODO data race here on ca_bundle if the user sets next_https_rescan_certs to true
.client = std.crypto.tls.Client.init(
tls.connection.stream_reader.interface(),
&tls.connection.stream_writer.interface,
.{
.host = .{ .explicit = remote_host },
.ca = .{ .bundle = client.ca_bundle },
.ca_bundle = &client.ca_bundle,
.ssl_key_log = client.ssl_key_log,
.read_buffer = tls_read_buffer,
.write_buffer = socket_write_buffer,
Expand Down Expand Up @@ -461,6 +459,20 @@ pub const Connection = struct {
}
};

fn caVerify(ca_bundle: *Certificate.Bundle, subject: Certificate.Parsed, now_sec: i64) Certificate.Bundle.VerifyError!void {
const client: *Client = @fieldParentPtr("ca_bundle", ca_bundle);

client.ca_bundle_mutex.lock();
defer client.ca_bundle_mutex.unlock();

client.ca_bundle.source = .system;
defer client.ca_bundle.source = .{ .callback = &caVerify };

// TODO: return the original error if it's part of VerifyError set
client.ca_bundle.verifyRescan(client.allocator, subject, now_sec) catch
return error.CertificateIssuerNotFound;
Comment on lines +471 to +473
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there's two ways to go here: a) return the VerifyError or CertificateIssuerNotFound for any rescan error, or b) add the rescan errors to VerifyError's set. I'm not sure what's preferable.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

3rd option: add error.CertificateBundleLoadFailure to VerifyError, and return that here instead of CertificateIssuerNotFound.

}

pub const Response = struct {
request: *Request,
/// Pointers in this struct are invalidated when the response body stream
Expand Down Expand Up @@ -1681,16 +1693,6 @@ pub fn request(

if (protocol == .tls) {
if (disable_tls) unreachable;
if (@atomicLoad(bool, &client.next_https_rescan_certs, .acquire)) {
client.ca_bundle_mutex.lock();
defer client.ca_bundle_mutex.unlock();

if (client.next_https_rescan_certs) {
client.ca_bundle.rescan(client.allocator) catch
return error.CertificateBundleLoadFailure;
@atomicStore(bool, &client.next_https_rescan_certs, false, .release);
}
}
}

const connection = options.connection orelse c: {
Expand Down