From b55588268d6a938c81df7fa1cd6b0ce154db04c8 Mon Sep 17 00:00:00 2001 From: sidey79 <7968127+sidey79@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:23:59 +0100 Subject: [PATCH] fix[build]: scan also META.json for additional needed CPAN packages. --- .github/workflows/build.yml | 15 ++-- scripts/get-Packages.pl | 2 +- scripts/parse-METAJson.pl | 136 ++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 scripts/parse-METAJson.pl diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f4d5db1..5e87acb7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,8 +38,8 @@ jobs: perl-version: "5.38" install-modules-with: cpanm install-modules-args: --notest - install-modules: PPI Perl::PrereqScanner::NotQuiteLite File::Find::Rule List::MoreUtils - + install-modules: PPI Perl::PrereqScanner::NotQuiteLite File::Find::Rule List::MoreUtils CPAN::Meta Module::CPANfile CPAN::Meta::Merge Scalar::Util + - name: clone 3rdparty repositories at github run: | @@ -55,17 +55,20 @@ jobs: - name: create cpanfile from local cloned 3rdparty repositories run: | scan-perl-prereqs-nqlite -save_cpanfile -suggests -private_re "^(FHEM::|Win32::|YAF$|OW$|RTypes$|RRDs$|SetExtensions$|HttpUtils$|UPnP::ControlPoint$|FritzBoxUtils$|configDB$|RESIDENTStk$|SHC_datafields$|TcpServerUtils$|Blocking$|uConv$|ZWLib$|UpNp:Common|HttpUtils$|Unit$|GD$|DevIo$|AttrTemplate$|ProtoThreads$|$FHEM_MODULES)" ./3rdparty - + perl scripts/parse-METAJson.pl ./3rdparty + - uses: actions/upload-artifact@v4 with: name: cpanfile-3rdParty path: cpanfile overwrite: true - - name: create cpanfile from FHEM dependencies + - name: create cpanfile from FHEM svn dependencies run: | + rm cpanfile scan-perl-prereqs-nqlite -save_cpanfile -suggests -private_re "^(FHEM::|Win32::|YAF$|OW$|RTypes$|RRDs$|SetExtensions$|HttpUtils$|UPnP::ControlPoint$|FritzBoxUtils$|configDB$|RESIDENTStk$|SHC_datafields$|TcpServerUtils$|Blocking$|uConv$|ZWLib$|UpNp:Common|HttpUtils$|Unit$|GD$|DevIo$|AttrTemplate$|ProtoThreads$|$FHEM_MODULES)" ./src/fhem/trunk - + perl scripts/parse-METAJson.pl ./src/fhem/trunk + - uses: actions/upload-artifact@v4 with: name: cpanfile-FHEM @@ -226,7 +229,7 @@ jobs: - name: Run build in unittests run: | - CONTAINER=$(docker run -d -ti --health-interval=10s --health-timeout=8s --health-start-period=10s --health-retries=5 -e CPAN_PKGS=Test2::Suite ${{ fromJSON(steps.meta.outputs.json).tags[0] }} ) + CONTAINER=$(docker run -d -ti --health-interval=10s --health-timeout=8s --health-start-period=10s --health-retries=5 ${{ fromJSON(steps.meta.outputs.json).tags[0] }} ) sleep 15; until [ "$(/usr/bin/docker inspect -f {{.State.Health.Status}} $CONTAINER)" == "healthy" ]; do sleep 1; diff --git a/scripts/get-Packages.pl b/scripts/get-Packages.pl index e920ded5..f004af1b 100755 --- a/scripts/get-Packages.pl +++ b/scripts/get-Packages.pl @@ -4,7 +4,7 @@ use PPI; use File::Find::Rule; use List::MoreUtils qw(uniq); -use Data::Dumper; +# use Data::Dumper; my @directories = @ARGV; diff --git a/scripts/parse-METAJson.pl b/scripts/parse-METAJson.pl new file mode 100644 index 00000000..3128740d --- /dev/null +++ b/scripts/parse-METAJson.pl @@ -0,0 +1,136 @@ +#!/bin/perl +use strict; +use warnings; +use CPAN::Meta; +use Module::CPANfile; +# use Data::Dumper; +use CPAN::Meta::Merge; +use Scalar::Util qw/blessed/; +use File::Find::Rule; +use JSON; + +my @directories = @ARGV; + +#my $filename = @ARGV # path must be provided to our script +my @JSONlines; +my $jsonString; +#my $line; +## Load existing requirements from cpanfile +my $cpanfile = Module::CPANfile->load('cpanfile'); + +my $newCPANFile; +# Alle Perl-Moduldateien im Verzeichnisbaum finden +foreach my $directory (@directories) { + + + my @files = File::Find::Rule->file()->name('*.pm')->in($directory); + + foreach my $filename (@files) { + $jsonString=""; + @JSONlines=(); + + print qq[\n try processing file $filename ...]; + open(my $fh, '<', $filename) or die "can not open file: $!"; + my $inside_for = 0; + + while (<$fh>) { + if (/^=for :application\/json;q=META.json/) { + $inside_for = 1; + next; + } + if ($inside_for) { + last if /^=end :application\/json;q=META.json/; # Ende des =for-Abschnitts + $_ =~ s/\R//g; + + push @JSONlines, $_; + + } + } + close($fh); + if (!@JSONlines) + { + print "aborting, no META.json found\n"; + next; + } + + $jsonString = join '', @JSONlines; + + ## Script breaks here, because we may have no version field which is requred to pass here + + my $MetaHash; + eval { + $MetaHash = from_json($jsonString) ; + 1; + } or do { + print q[[ failed ]]. $@; + next; + }; + + #print Dumper $MetaHash; + + # fix missing version information + my @fixups = (); + + if (!exists($MetaHash->{version}) ) + { + push(@fixups, q[version] ); + + $MetaHash->{version} = "1"; + + } + if (!exists($MetaHash->{name}) ) + { + push(@fixups, q[name] ); + $MetaHash->{name} = $filename; + } + + if (!exists($MetaHash->{'meta-spec'}) || !exists($MetaHash->{'meta-spec'}{'version'}) ) + { + push(@fixups, q[meta-spec] ); + $MetaHash->{'meta-spec'}{'version'} = 2; + } + + if (scalar @fixups > 0) + { + print q[ fixups: ]; + print join ", ", @fixups; + print q[ ]; + } + + + my $moduleMeta; + eval { + $moduleMeta = CPAN::Meta->load_json_string(to_json($MetaHash)); + 1; + } or do { + print q[[ failed ]]. $@; + next; + }; + + + # merge requirements + + my $prereqs_hash = $cpanfile->prereqs->with_merged_prereqs($moduleMeta->effective_prereqs)->as_string_hash; + my $struct = { %{$moduleMeta->as_struct}, prereqs => $prereqs_hash }; + + #print $moduleMeta->meta_spec->{version}; + + my $mergedMeta = CPAN::Meta->new($struct); + $newCPANFile = Module::CPANfile->from_prereqs( $mergedMeta->prereqs ); + + $cpanfile = $newCPANFile; + print qq[$filename was processed successfull\n]; + } +} + +# save our new cpanfile +if (defined $newCPANFile) +{ + $newCPANFile->save('cpanfile'); + print qq[\n\nResult: cpanfile was saved\n]; +} + + +# example usage +# perl scripts/parse-METAJson.pl src/FHEM/trunk/fhem/FHEM/ 3rdParty/ +# \ No newline at end of file