[portgen go] Fix issues with missing dependencies

classic Classic list List threaded Threaded
3 messages Options
Reply | Threaded
Open this post in threaded view
|

[portgen go] Fix issues with missing dependencies

Aaron Bieber-2
Hola,

This change addresses a few issues, the most important being missing
dependencies. Now we use 'go list -m all' to get a list of "hard" deps
(hard meaning we need the actual code [.zip files] to build the
package).

The list created by 'go mod graph' is now pruned to exclude the list
from 'go list' and is treated as the source of .mod files (needed for go
to re-calculate the build dependencies).

It also:
   - De-duplicates the MODGO_MOD* lists so Makefiles are no longer 3k
     lines long!
   - Puts the chdir stuff into a child process (requested by espie@).
   - Puts MODGO_MOD{ULES,FILES} values on a new line so less space is
     used (also requested by espie@).
   - Reduces the quality of the perl used (sorry afresh1@ <3).

Now things like 'github.com/gruntwork-io/terragrunt' and
'github.com/coredns/coredns` are able to be portgen'd!

OK?

Cheers,
Aaron

diff --git a/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
index 96e81252dcf..5be497d95f1 100644
--- a/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
+++ b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
@@ -91,43 +91,16 @@ sub get_dist_info
  my ( $self, $module ) = @_;
 
  my $json = $self->_go_determine_name($module);
-
- my %mods;
- for ( $self->_go_mod_graph($json) ) {
- my ($direct, $ephemeral) = @{$_};
-
- for my $d ( $direct, $ephemeral ) {
- next unless $d->{Version};
- $mods{ $d->{Module} }{ $d->{Version} } ||= $d;
- }
- }
-
- # Filter dependencies to the one with the highest version
- foreach my $mod ( sort keys %mods ) {
- # Sort semver numerically then the rest alphanumeric.
- # This is overkill for sorting, but I already wrote it
- my @versions =
-    map { $_->[0] }
-    sort {
-        $a->[1] <=> $b->[1]
-     || $a->[2] <=> $b->[2]
-     || $a->[3] <=> $b->[3]
-     || $a->[4] cmp $b->[4]
-    }
-    map { $_->[1] =~ s/^v//; $_ }
-    map { [ $_, split /[\.\+-]/, $_, 4 ] }
-    keys %{ $mods{$mod} };
-
- push @{ $json->{Dist} }, $mods{$mod}{ $versions[-1] };
- push @{ $json->{Mods} }, map { $mods{$mod}{$_} } @versions;
- }
-
+ my ($dist, $mods) = $self->_go_mod_info($json);
  $json->{License} = $self->_go_lic_info($module);
 
+ $json->{Dist} = $dist if @$dist > 0;
+ $json->{Mods} = $mods if @$mods > 0;
+
  return $json;
 }
 
-sub _go_mod_graph
+sub _go_mod_info
 {
  my ($self, $json) = @_;
  my $dir = tempdir(CLEANUP => 0);
@@ -138,7 +111,6 @@ sub _go_mod_graph
  #$module =~ s/\s+$//;
  unless ( $json->{Module} eq $module ) {
  my $msg = "Module $json->{Module} doesn't match $module";
- warn "$msg\n";
  croak $msg;
  }
 
@@ -148,33 +120,76 @@ sub _go_mod_graph
  close $fh;
  }
 
- my $old_cwd = getcwd();
- chdir $dir or die "Unable to chdir '$dir': $!";
+ my @deps;
+ my $fh;
+ my $pid = open($fh, "-|");
+ if (!defined $pid) {
+ die "unable to fork";
+ }
+ if ($pid == 0) {
+ chdir $dir or die "Unable to chdir '$dir': $!";
+ $ENV{GOPATH} = "$dir/go";
+ # Outputs: "dep version"
+ exec (qw(go list -m all));
+ }
+ my @raw_deps = <$fh>;
+ chomp @raw_deps;
+
+ close $fh or die "Unable to close pipe 'go list': $!";
+
+ @deps = map { $self->_go_mod_normalize( @{ $_ } ) }
+ grep { $_->[1] } map { [ split / / ] } # module space version
+ @raw_deps;
 
  my @mods;
- {
- # Outputs: "direct_dep ephemeral_dep"
- local $ENV{GOPATH} = "$dir/go";
- open my $fh, '-|', qw< go mod graph >
-    or die "Unable to spawn 'go mod path': $!";
- @mods = readline $fh;
- close $fh
-    or die "Error closing pipe to 'go mod path': $!";
+ $pid = open($fh, "-|");
+ if (!defined $pid) {
+ die "unable to fork";
+ }
+ if ($pid == 0) {
+ chdir $dir or die "Unable to chdir '$dir': $!";
+ $ENV{GOPATH} = "$dir/go";
+ # Outputs: "dep@version subdep@version"
+ exec (qw(go mod graph));
  }
+ my @all_mods = <$fh>;
+ chomp @all_mods;
+
+ close $fh or die "Unable to close pipe 'go mod': $!";
 
- chdir $old_cwd or die "Unable to chdir '$old_cwd': $!";
+ my %all_deps = map { $_ => 1 } @deps;
+ push @mods, grep { !$all_deps{$_}++ }
+ map { $self->_go_mod_normalize( @{ $_ } ) }
+ grep { $_->[1] } map { [ split /\@/ ] } # module @ version
+ map { split /\s+/ } @all_mods;
 
- chomp @mods;
 
- # parse the graph into pairs of hashrefs
- return map { [
-    map {
-        my ($m, $v) = split /@/;
-        { Module => $m, Version => $v };
-    } split /\s/
- ] } grep { $_ } @mods;
+ foreach my $fl ( \@deps, \@mods ) {
+ next unless @$fl > 0; # if there aren't any, don't try
+ my @s = map {
+ my @f = split(/ /, $_);
+ [$f[0], $f[1]];
+ } @$fl;
+ my ($length) = sort { $b <=> $a } map { length $_->[0] } @s;
+ my $n = ( 1 + int $length / 8 );
+ @s = map {
+ my $tabs = "\t" x ( $n - int( length($_->[0]) / 8 ) );
+ "\t$_->[0]$tabs $_->[1]"
+ } @s;
+ @$fl = @s;
+ }
+
+ return ( \@deps, \@mods );
+}
+
+sub _go_mod_normalize
+{
+ my ( $self, $module, $version ) = @_;
+ ( my $l = $module ) =~ s/\p{Upper}/!\L$&/g;
+ return "$l $version";
 }
 
+
 sub get_ver_info
 {
  my ( $self, $module ) = @_;
@@ -236,25 +251,8 @@ sub fill_in_makefile
  $self->set_pkgname($di->{Name} . "-" . $parts[0]);
  }
 
- my @dist = map { [ $_->{Module}, $_->{Version} ] }
-    @{ $di->{Dist} || [] };
- my @mods = map { [ $_->{Module}, $_->{Version} ] }
-    @{ $di->{Mods} };
-
- # Turn the deps into tab separated columns
- foreach my $s ( \@dist, \@mods ) {
- next unless @{$s}; # if there aren't any, don't try
- my ($length) = sort { $b <=> $a } map { length $_->[0] } @$s;
- my $n = ( 1 + int $length / 8 );
- @{$s} = map {
-    ( my $l = $_->[0] ) =~ s/\p{Upper}/!\L$&/g;
-    my $tabs = "\t" x ( $n - int( length($l) / 8 ) );
-    "$l$tabs $_->[1]"
- } @{$s};
- }
-
- $self->set_other( MODGO_MODULES  => \@dist ) if @dist;
- $self->set_other( MODGO_MODFILES => \@mods ) if @mods;
+ $self->set_other( MODGO_MODULES  => "\\\n" . join(" \\\n", @{$di->{Dist}})) if $di->{Dist};
+ $self->set_other( MODGO_MODFILES => "\\\n" . join(" \\\n", @{$di->{Mods}})) if $di->{Mods};
 }
 
 sub try_building
diff --git a/lang/go/go.port.mk b/lang/go/go.port.mk
index a432e157b80..9db9faac652 100644
--- a/lang/go/go.port.mk
+++ b/lang/go/go.port.mk
@@ -65,6 +65,7 @@ EXTRACT_ONLY = ${DISTNAME}${EXTRACT_SUFX}
 MASTER_SITES ?= ${MASTER_SITE_ATHENS}${MODGO_MODNAME}/@v/
 .  for _modpath _modver in ${MODGO_MODULES}
 DISTFILES += ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.zip{${_modpath}/@v/${_modver}.zip}:${MODGO_MASTER_SITESN}
+DISTFILES += ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.mod{${_modpath}/@v/${_modver}.mod}:${MODGO_MASTER_SITESN}
 .  endfor
 .  for _modpath _modver in ${MODGO_MODFILES}
 DISTFILES += ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.mod{${_modpath}/@v/${_modver}.mod}:${MODGO_MASTER_SITESN}

--
PGP: 0x1F81112D62A9ADCE / 3586 3350 BFEA C101 DB1A  4AF0 1F81 112D 62A9 ADCE

Reply | Threaded
Open this post in threaded view
|

Re: [portgen go] Fix issues with missing dependencies

Aaron Bieber-2

Aaron Bieber writes:

> Hola,
>
> This change addresses a few issues, the most important being missing
> dependencies. Now we use 'go list -m all' to get a list of "hard" deps
> (hard meaning we need the actual code [.zip files] to build the
> package).
>
> The list created by 'go mod graph' is now pruned to exclude the list
> from 'go list' and is treated as the source of .mod files (needed for go
> to re-calculate the build dependencies).
>
> It also:
>    - De-duplicates the MODGO_MOD* lists so Makefiles are no longer 3k
>      lines long!
>    - Puts the chdir stuff into a child process (requested by espie@).
>    - Puts MODGO_MOD{ULES,FILES} values on a new line so less space is
>      used (also requested by espie@).
>    - Reduces the quality of the perl used (sorry afresh1@ <3).
>
> Now things like 'github.com/gruntwork-io/terragrunt' and
> 'github.com/coredns/coredns` are able to be portgen'd!
>
> OK?
>
> Cheers,
> Aaron
>
Previous version suffered from some issues. It didn't handle redirects (
https://github.com/gohugoio/hugo/blob/master/go.mod#L71 ) and the map{}
bits were hard to debug.

This version reduces the # of lines in www/hugo from 658 to 403!


diff --git a/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
index 96e81252dcf..4089df3edc9 100644
--- a/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
+++ b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
@@ -91,90 +91,124 @@ sub get_dist_info
  my ( $self, $module ) = @_;
 
  my $json = $self->_go_determine_name($module);
+ my ($dist, $mods) = $self->_go_mod_info($json);
+ $json->{License} = $self->_go_lic_info($module);
 
- my %mods;
- for ( $self->_go_mod_graph($json) ) {
- my ($direct, $ephemeral) = @{$_};
+ $json->{Dist} = $dist if @$dist > 0;
+ $json->{Mods} = $mods if @$mods > 0;
 
- for my $d ( $direct, $ephemeral ) {
- next unless $d->{Version};
- $mods{ $d->{Module} }{ $d->{Version} } ||= $d;
- }
- }
+ return $json;
+}
+
+sub _run
+{
+ my ($self, $cmd, $dir) = @_;
+ my $fh;
 
- # Filter dependencies to the one with the highest version
- foreach my $mod ( sort keys %mods ) {
- # Sort semver numerically then the rest alphanumeric.
- # This is overkill for sorting, but I already wrote it
- my @versions =
-    map { $_->[0] }
-    sort {
-        $a->[1] <=> $b->[1]
-     || $a->[2] <=> $b->[2]
-     || $a->[3] <=> $b->[3]
-     || $a->[4] cmp $b->[4]
-    }
-    map { $_->[1] =~ s/^v//; $_ }
-    map { [ $_, split /[\.\+-]/, $_, 4 ] }
-    keys %{ $mods{$mod} };
-
- push @{ $json->{Dist} }, $mods{$mod}{ $versions[-1] };
- push @{ $json->{Mods} }, map { $mods{$mod}{$_} } @versions;
+ my $pid = open($fh, "-|");
+ if (!defined $pid) {
+ die "unable to fork";
  }
 
- $json->{License} = $self->_go_lic_info($module);
+ if ($pid == 0) {
+ chdir $dir or die "Unable to chdir '$dir': $!";
+ $ENV{GOPATH} = "$dir/go";
+ $ENV{GO111MODULE} = "on";
+ # Outputs: "dep version"
+ exec ($cmd);
+ die "exec didn't work: $!";
+ }
 
- return $json;
+ my @output = <$fh>;
+ chomp @output;
+ close $fh or die "Unable to close pipe '$cmd': $!";
+ return @output;
 }
 
-sub _go_mod_graph
+sub _go_mod_info
 {
  my ($self, $json) = @_;
  my $dir = tempdir(CLEANUP => 0);
 
  my $mod = $self->get("$json->{Module}/\@v/$json->{Version}.mod");
  my ($module) = $mod =~ /\bmodule\s+(.*?)\s/;
- #my ($module) = $mod =~ /\bmodule\s+(.*)\n/;
- #$module =~ s/\s+$//;
+
  unless ( $json->{Module} eq $module ) {
  my $msg = "Module $json->{Module} doesn't match $module";
- warn "$msg\n";
  croak $msg;
  }
 
- {
- open my $fh, '>', $dir . "/go.mod" or die $!;
- print $fh $mod;
- close $fh;
+ open my $fh, '>', $dir . "/go.mod" or die $!;
+ print $fh $mod;
+ close $fh;
+
+ # Outputs: "dep version"
+ my @raw_deps = $self->_run("go list -m all", $dir);
+ my @deps;
+ my $all_deps = {};
+ foreach my $dep (@raw_deps) {
+ next if $dep eq $json->{Module};
+ if ($dep =~ m/=>/) {
+ foreach my $d (split(/ => /, $dep)) {
+ my $smod = $self->_go_mod_normalize($d);
+ push @deps, $smod unless defined $all_deps->{$smod};
+ $all_deps->{$smod} = 1;
+ }
+ } else {
+ my $nmod = $self->_go_mod_normalize($dep);
+ push @deps, $nmod unless defined $all_deps->{$nmod};
+ $all_deps->{$nmod} = 1;
+ }
  }
 
- my $old_cwd = getcwd();
- chdir $dir or die "Unable to chdir '$dir': $!";
-
+ # Outputs: "dep@version subdep@version"
+ my @raw_mods =  $self->_run("go mod graph", $dir);
  my @mods;
- {
- # Outputs: "direct_dep ephemeral_dep"
- local $ENV{GOPATH} = "$dir/go";
- open my $fh, '-|', qw< go mod graph >
-    or die "Unable to spawn 'go mod path': $!";
- @mods = readline $fh;
- close $fh
-    or die "Error closing pipe to 'go mod path': $!";
+
+ foreach my $mod (@raw_mods) {
+ carp Dumper $mod if ($mod =~ m/markbates/);
+ foreach my $m (split(/ /, $mod)) {
+ $m =~ s/@/ /;
+ $m = $self->_go_mod_normalize($m);
+ if (! defined $all_deps->{$m}) {
+ push @mods, $m unless $m eq $json->{Module};
+ $all_deps->{$m} = 1;
+ }
+ }
  }
 
- chdir $old_cwd or die "Unable to chdir '$old_cwd': $!";
+ foreach my $fl ( \@deps, \@mods ) {
+ next unless @$fl > 0; # if there aren't any, don't try
+ my @s = map {
+ my @f = split(/ /, $_);
+ [$f[0], $f[1]];
+ } @$fl;
+ my ($length) = sort { $b <=> $a } map { length $_->[0] } @s;
+ my $n = ( 1 + int $length / 8 );
+ @s = map {
+ my $tabs = "\t" x ( $n - int( length($_->[0]) / 8 ) );
+ "\t$_->[0]$tabs $_->[1]"
+ } @s;
+ @$fl = @s;
+ }
+
+
+ @deps = sort @deps;
+ @mods = sort @mods;
 
- chomp @mods;
+ return ( \@deps, \@mods );
+}
 
- # parse the graph into pairs of hashrefs
- return map { [
-    map {
-        my ($m, $v) = split /@/;
-        { Module => $m, Version => $v };
-    } split /\s/
- ] } grep { $_ } @mods;
+sub _go_mod_normalize
+{
+ my ( $self, $line) = @_;
+ chomp $line;
+ $line =~ s/\p{Upper}/!\L$&/g;
+ $line =~ s/\s+/ /g;
+ return $line;
 }
 
+
 sub get_ver_info
 {
  my ( $self, $module ) = @_;
@@ -236,25 +270,8 @@ sub fill_in_makefile
  $self->set_pkgname($di->{Name} . "-" . $parts[0]);
  }
 
- my @dist = map { [ $_->{Module}, $_->{Version} ] }
-    @{ $di->{Dist} || [] };
- my @mods = map { [ $_->{Module}, $_->{Version} ] }
-    @{ $di->{Mods} };
-
- # Turn the deps into tab separated columns
- foreach my $s ( \@dist, \@mods ) {
- next unless @{$s}; # if there aren't any, don't try
- my ($length) = sort { $b <=> $a } map { length $_->[0] } @$s;
- my $n = ( 1 + int $length / 8 );
- @{$s} = map {
-    ( my $l = $_->[0] ) =~ s/\p{Upper}/!\L$&/g;
-    my $tabs = "\t" x ( $n - int( length($l) / 8 ) );
-    "$l$tabs $_->[1]"
- } @{$s};
- }
-
- $self->set_other( MODGO_MODULES  => \@dist ) if @dist;
- $self->set_other( MODGO_MODFILES => \@mods ) if @mods;
+ $self->set_other( MODGO_MODULES  => "\\\n" . join(" \\\n", @{$di->{Dist}})) if $di->{Dist};
+ $self->set_other( MODGO_MODFILES => "\\\n" . join(" \\\n", @{$di->{Mods}})) if $di->{Mods};
 }
 
 sub try_building
diff --git a/lang/go/go.port.mk b/lang/go/go.port.mk
index a432e157b80..9db9faac652 100644
--- a/lang/go/go.port.mk
+++ b/lang/go/go.port.mk
@@ -65,6 +65,7 @@ EXTRACT_ONLY = ${DISTNAME}${EXTRACT_SUFX}
 MASTER_SITES ?= ${MASTER_SITE_ATHENS}${MODGO_MODNAME}/@v/
 .  for _modpath _modver in ${MODGO_MODULES}
 DISTFILES += ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.zip{${_modpath}/@v/${_modver}.zip}:${MODGO_MASTER_SITESN}
+DISTFILES += ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.mod{${_modpath}/@v/${_modver}.mod}:${MODGO_MASTER_SITESN}
 .  endfor
 .  for _modpath _modver in ${MODGO_MODFILES}
 DISTFILES += ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.mod{${_modpath}/@v/${_modver}.mod}:${MODGO_MASTER_SITESN}


--
PGP: 0x1F81112D62A9ADCE / 3586 3350 BFEA C101 DB1A  4AF0 1F81 112D 62A9 ADCE
Reply | Threaded
Open this post in threaded view
|

Re: [portgen go] Fix issues with missing dependencies

Marc Espie-2
On Wed, Jun 24, 2020 at 09:38:38PM -0600, Aaron Bieber wrote:
> + my @output = <$fh>;
> + chomp @output;
> + close $fh or die "Unable to close pipe '$cmd': $!";

You still missed that one: you won't get anything out of $!, it's $?
that's gonna hold the child exit status.