From f9c854aba26c4df5722f130acd847e0f6668a5db Mon Sep 17 00:00:00 2001 From: Brian Fraser Date: Mon, 23 Nov 2020 11:40:40 +0100 Subject: [PATCH 1/2] Makefile.PL: optionally use Alien::librdkafka This allows installing Net::Kafka even when there is no librdkafka provided by the OS itself. This was added as a `recommends` dependency for the configure step, so if it is missing we try installing Net::Kafka just like we did before. --- Makefile.PL | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/Makefile.PL b/Makefile.PL index 3ee8edf..47145b9 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -8,11 +8,62 @@ use ExtUtils::PkgConfig; use constant LIBRDKAFKA_MIN_VERSION => "1.0.0"; +# Fix for OSX: +$ENV{PKG_CONFIG_PATH} .= ':/usr/local/opt/openssl/lib/pkgconfig/' + if $^O eq 'darwin'; +# rdkafka.pc has libcrypto as a required dependency, so +# if libcrypto can't be found, pkg-config won't return +# values for rdkafka. +# Problem: in OSX, the default libcrypto lives in a non-default +# location, meaning that without some tweaking like above, +# we never get values for rdkafka. + +my %makemaker_args = ( + LIBS => ['-lpthread -lrdkafka'], +); + # If you have a local librdkafka installation and you want to use that instead # of the system-wide, do something like: # PKG_CONFIG_PATH=/usr/local/lib/pkgconfig perl Makefile.PL # or whatever is the actual installation path -my %rdkafka = ExtUtils::PkgConfig->find('rdkafka'); +my %rdkafka; +eval { + %rdkafka = ExtUtils::PkgConfig->find('rdkafka'); + 1; +} or do { + my $pkg_config_error = $@ || 'Zombie error'; + + eval { + require Alien::librdkafka; + my $dist_dir = Alien::librdkafka->dist_dir; + local $ENV{PKG_CONFIG_PATH} = $ENV{PKG_CONFIG_PATH} // ''; + # TODO: broken for Windows + $ENV{PKG_CONFIG_PATH} .= ':' . File::Spec->catdir($dist_dir, qw); + # Try again, first for the static version (since linking to the + # shared library when it comes from some Perl package will make it + # unreachable at runtime) + eval { + %rdkafka = ExtUtils::PkgConfig->find('rdkafka-static'); + $makemaker_args{LDDLFLAGS} = join ' ', + $Config{lddlflags}, + $rdkafka{libs}, + ; + $makemaker_args{LIBS} = ['-lpthread']; + 1; + } or do { + %rdkafka = ExtUtils::PkgConfig->find('rdkafka'); + }; + 1; + } or do { + my $alien_librdkafka_error = $@ || 'Zombie error'; + die "pkg-config nor Alien::librdkafka could find kafka; bailing out!\n" + ."PkgConfig error: $pkg_config_error\n" + ."Alien::librdkafka error: $alien_librdkafka_error\n" + ; + }; +}; + +$makemaker_args{INC} = $rdkafka{cflags}; warn sprintf( "WARNING: Installed librdkafka version %s is lower than tested %s", @@ -40,7 +91,7 @@ WriteMakefile( LICENSE => 'perl', VERSION_FROM => 'lib/Net/Kafka.pm', ABSTRACT_FROM => 'lib/Net/Kafka.pm', - LIBS => [ "-lrdkafka -lpthread" ], + %makemaker_args, OBJECT => '$(O_FILES)', TYPEMAPS => ['typemap'], PREREQ_PM => { @@ -50,6 +101,14 @@ WriteMakefile( }, META_MERGE => { "meta-spec" => { version => 2 }, + prereqs => { + configure => { + # optionally use Alien::librdkafka + recommends => { + 'Alien::librdkafka' => '0.01', + }, + }, + }, resources => { homepage => 'https://github.com/bookingcom/perl-Net-Kafka', bugtracker => { From ed1bae8c987e8634effcde78511e5cd1897e92f4 Mon Sep 17 00:00:00 2001 From: Brian Fraser Date: Tue, 1 Dec 2020 15:56:02 +0100 Subject: [PATCH 2/2] Improvements to the Alien::librdkafka codepath --- Makefile.PL | 99 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/Makefile.PL b/Makefile.PL index 47145b9..8b3201e 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -5,6 +5,7 @@ use warnings; use Config; use ExtUtils::MakeMaker; use ExtUtils::PkgConfig; +use Text::ParseWords qw(shellwords); use constant LIBRDKAFKA_MIN_VERSION => "1.0.0"; @@ -18,41 +19,78 @@ $ENV{PKG_CONFIG_PATH} .= ':/usr/local/opt/openssl/lib/pkgconfig/' # location, meaning that without some tweaking like above, # we never get values for rdkafka. -my %makemaker_args = ( - LIBS => ['-lpthread -lrdkafka'], -); +sub prepare_lddlflags { + my ($lddlflags) = @_; + # `-Wl,-z,now` breaks static linking to librdkafka.a, and + # the perl you get from pacman in ArchLinux has that by default + # in lddlflags. + # Similar with --as-needed + $lddlflags =~ s<,-z[,= ]now><>g; + $lddlflags =~ s<,--as-needed\b><>g; + $lddlflags =~ s<-Wl\s>< >g; # did we remove every linker option? + return $lddlflags; +} + +my @archive_after; +my $makemaker_libs = []; +my $makemaker_inc = ''; + +my $ccflags = $Config{'ccflags'} // ''; +my $lddlflags = prepare_lddlflags($Config{'lddlflags'} // ''); # If you have a local librdkafka installation and you want to use that instead # of the system-wide, do something like: # PKG_CONFIG_PATH=/usr/local/lib/pkgconfig perl Makefile.PL # or whatever is the actual installation path -my %rdkafka; eval { - %rdkafka = ExtUtils::PkgConfig->find('rdkafka'); + my %rdkafka = ExtUtils::PkgConfig->find('rdkafka'); + $makemaker_libs = '-lpthread -lrdkafka'; + $lddlflags .= ' ' . $rdkafka{libs}; + $makemaker_inc = $rdkafka{cflags}; + warn sprintf( + "WARNING: Installed librdkafka version %s is lower than tested %s", + $rdkafka{modversion}, + LIBRDKAFKA_MIN_VERSION + ) if $rdkafka{modversion} lt LIBRDKAFKA_MIN_VERSION; + + print "Compiling Net::Kafka with librdkafka $rdkafka{modversion}\n\n"; + 1; } or do { my $pkg_config_error = $@ || 'Zombie error'; eval { require Alien::librdkafka; - my $dist_dir = Alien::librdkafka->dist_dir; - local $ENV{PKG_CONFIG_PATH} = $ENV{PKG_CONFIG_PATH} // ''; - # TODO: broken for Windows - $ENV{PKG_CONFIG_PATH} .= ':' . File::Spec->catdir($dist_dir, qw); - # Try again, first for the static version (since linking to the - # shared library when it comes from some Perl package will make it - # unreachable at runtime) - eval { - %rdkafka = ExtUtils::PkgConfig->find('rdkafka-static'); - $makemaker_args{LDDLFLAGS} = join ' ', - $Config{lddlflags}, - $rdkafka{libs}, - ; - $makemaker_args{LIBS} = ['-lpthread']; - 1; - } or do { - %rdkafka = ExtUtils::PkgConfig->find('rdkafka'); - }; + $makemaker_inc .= ' ' . Alien::librdkafka->cflags; + my $libs = Alien::librdkafka->install_type eq 'share' + ? Alien::librdkafka->libs_static || Alien::librdkafka->libs + : Alien::librdkafka->libs + ; + # cmake/OSX bug: -lz gets transformed into + # -l/abs/path/to/libz.tbd + # in systems with Apple's "built-in dynamic linker cache" + $libs =~ s<\B-l/>g; + + my $lib_ext = $Config{lib_ext}; + foreach my $maybe_lib ( shellwords($libs) ) { + if ( $maybe_lib =~ m/\B-l(\S+)/ ) { + # -lfoo + push @$makemaker_libs, $maybe_lib; + } + elsif ( $maybe_lib =~ m<-L(\S+)> ) { + my $lib_dir = $1; + # /path/to/libs + # add '-Wl,-rpath=$dir' + $libs = '-Wl,-rpath,' . $lib_dir . ' ' . $libs; + } + elsif ( $maybe_lib =~ m<^/.+$lib_ext$> ) { + # .a file in pkg-config --libs output -- this needs to come after the -o + push @archive_after, $maybe_lib; + } + } + + $lddlflags .= ' ' . $libs; + 1; } or do { my $alien_librdkafka_error = $@ || 'Zombie error'; @@ -63,17 +101,13 @@ eval { }; }; -$makemaker_args{INC} = $rdkafka{cflags}; - -warn sprintf( - "WARNING: Installed librdkafka version %s is lower than tested %s", - $rdkafka{modversion}, - LIBRDKAFKA_MIN_VERSION -) if $rdkafka{modversion} lt LIBRDKAFKA_MIN_VERSION; - -print "Compiling Net::Kafka with librdkafka $rdkafka{modversion}\n\n"; WriteMakefile( + INC => $makemaker_inc, + LIBS => $makemaker_libs, + LDDLFLAGS => $lddlflags, + CCFLAGS => $ccflags, + PERL_ARCHIVE_AFTER => join(' ', @archive_after), NAME => 'Net::Kafka', DISTNAME => 'Net-Kafka', AUTHOR => [ @@ -91,7 +125,6 @@ WriteMakefile( LICENSE => 'perl', VERSION_FROM => 'lib/Net/Kafka.pm', ABSTRACT_FROM => 'lib/Net/Kafka.pm', - %makemaker_args, OBJECT => '$(O_FILES)', TYPEMAPS => ['typemap'], PREREQ_PM => {