[csw-devel] SF.net SVN: opencsw:[612] buildfarm/bin

[dmichelsen at users.sourceforge.net]

Revision: 612
          http://opencsw.svn.sourceforge.net/opencsw/?rev=612&view=rev
Author:   dmichelsen
Date:     2012-02-22 20:42:05 +0000 (Wed, 22 Feb 2012)
Log Message:
-----------
Add REST catalog interfacing

Added Paths:
-----------
    buildfarm/bin/rest-interface/
    buildfarm/bin/rest-interface/catalogname4pkg
    buildfarm/bin/rest-interface/getcatalog

Added: buildfarm/bin/rest-interface/catalogname4pkg
===================================================================
--- buildfarm/bin/rest-interface/catalogname4pkg	                        (rev 0)
+++ buildfarm/bin/rest-interface/catalogname4pkg	2012-02-22 20:42:05 UTC (rev 612)
@@ -0,0 +1,42 @@
+#!/opt/csw/bin/perl
+
+use warnings;
+use strict;
+use LWP::Simple;
+use JSON;
+use Data::Dumper;
+
+require LWP::UserAgent;
+
+my $ua = LWP::UserAgent->new;
+$ua->timeout(300);
+
+my $catalog = "unstable";
+my $arch = "sparc";
+my $rel = "SunOS5.10";
+my $pkg = shift @ARGV;
+
+my $url = "http://buildfarm.opencsw.org/pkgdb/rest/catalogs/$catalog/$arch/$rel/pkgnames/$pkg/";
+
+print $url, "\n";
+
+my $response = $ua->get( $url );
+
+my $json_string;
+if ($response->is_success) {
+  $json_string = $response->decoded_content;  # or whatever
+} else {
+  die "Error getting $url\n" . $response->status_line;
+}
+
+if( $json_string eq "null" ) {
+  print "Paket $pkg konnte nicht gefunden werden in $catalog:$arch:$rel\n";
+  exit( 1 );
+}
+
+print "Decoding JSON... $json_string\n";
+my $data = decode_json $json_string;
+
+my $cname = $data->{catalogname};
+
+print "$cname\n";


Property changes on: buildfarm/bin/rest-interface/catalogname4pkg
___________________________________________________________________
Added: svn:executable
   + *

Added: buildfarm/bin/rest-interface/getcatalog
===================================================================
--- buildfarm/bin/rest-interface/getcatalog	                        (rev 0)
+++ buildfarm/bin/rest-interface/getcatalog	2012-02-22 20:42:05 UTC (rev 612)
@@ -0,0 +1,118 @@
+#!/opt/csw/bin/perl
+
+use strict;
+use warnings;
+
+use LWP::Simple;
+use JSON;
+use DB_File;
+use Data::Dumper;
+use Parallel::ForkManager;
+use LWP::UserAgent;
+
+my $ua = LWP::UserAgent->new;
+$ua->timeout(600);
+
+my %md5cache;
+
+my $db = tie %md5cache, "DB_File", "pkgcache.dbfile";
+
+print "Grabbing catalog...\n";
+my $response = $ua->get( 'http://buildfarm.opencsw.org/pkgdb/rest/catalogs/unstable/sparc/SunOS5.10/?quick=true' );
+my $json_string;
+if ($response->is_success) {
+print "Decoding JSON...\n";
+  $json_string = $response->decoded_content;  # or whatever
+} else {
+  print Dumper( $response );
+  die $response->status_line;
+}
+
+#my $url = 'http://buildfarm.opencsw.org/pkgdb/rest/catalogs/unstable/sparc/SunOS5.10/';
+#my $json_string = get $url;
+# die "Couldn't get $url" unless defined $json_string;
+my $catalog_data = decode_json $json_string;
+
+# print Dumper($catalog_data);
+
+#$VAR1 = {
+#          'catalogname' => 'dovecot_dev',
+#          'version_string' => '2.0.17,REV=2012.01.20',
+#          'vendor_url' => 'http://dovecot.org/',
+#          'md5_sum' => '62713e60bc4d8f8c895c11db84942c67',
+#          'arch' => 'sparc',
+#          'osrel' => 'SunOS5.9',
+#          'size' => 188321,
+#          'file_basename' => 'dovecot_dev-2.0.17,REV=2012.01.20-SunOS5.9-sparc-CSW.pkg.gz',
+#          'maintainer_email' => 'jgoerzen at opencsw.org',
+#          'maintainer_id' => 15,
+#          'version' => '2.0.17,REV=2012.01.20',
+#          'mtime' => '2012-01-20 19:26:57',
+#          'pkgname' => 'CSWdovecot-dev',
+#          'repository_url' => 'https://gar.svn.sourceforge.net/svnroot/gar/csw/mgar/pkg/dovecot/trunk/Makefile',
+#          'basename' => 'dovecot_dev-2.0.17,REV=2012.01.20-SunOS5.9-sparc-CSW.pkg.gz',
+#          'maintainer_full_name' => 'Jake Goerzen',
+#          'filename_arch' => 'sparc',
+#          'rev' => '2012.01.20'
+#        };
+
+
+my $pm = new Parallel::ForkManager(10, '/home/web/rest-interface/tmp' );
+
+# data structure retrieval and handling
+$pm -> run_on_finish ( # called BEFORE the first call to start()
+  sub {
+    my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_;
+
+    # retrieve data structure from child
+    if (defined($data_structure_reference)) {
+      my ($md5, $stats) = @{$data_structure_reference};
+      $md5cache{$md5} = $stats;
+      $db->sync;
+    } else {
+      print qq|No message received from child process $pid!\n|;
+    }
+  }
+);
+
+# Retreive the missing data
+foreach my $pkg (@$catalog_data) {
+  my $md5 = $pkg->{md5_sum};
+
+  my $url = "http://buildfarm.opencsw.org/pkgdb/rest/srv4/${md5}/pkg-stats/";
+
+  if( exists $md5cache{$md5} ) {
+    print "Having $md5\n";
+  } else {
+    $pm->start() and next;
+
+    print "Retreiving $md5\n";
+    my $stats = get $url;
+    print "Done $md5\n";
+    $pm->finish( 0, [ $md5, $stats ] );
+  }
+}
+
+$pm->wait_all_children;
+
+open( C, ">catalog.new" );
+
+foreach my $pkg (@$catalog_data) {
+  my $md5 = $pkg->{md5_sum};
+  my $stats = $md5cache{$md5};
+  my $json_stats = decode_json( $stats );
+
+  my @depends = map { $_->[0] } @{$json_stats->{depends}};
+  my @i_depends = @{$json_stats->{i_depends}};
+
+
+#  print C join( ' ', $pkg->{catalogname}, $pkg->{version}, $pkg->{pkgname}, $pkg->{file_basename}, $pkg->{md5_sum}, $pkg->{size}, 
+  print C join( ' ', $pkg->{catalogname}, $pkg->{version}, $json_stats->{pkginfo}->{PKG}, $pkg->{file_basename}, $pkg->{md5_sum}, $pkg->{size}, 
+    join( '|', @depends ) || 'none', join( '|', @i_depends ) || 'none', 'none', $json_stats->{pkginfo}->{OPENCSW_BUNDLE} || 'none' ), "\n";
+}
+
+close C;
+
+unlink( "catalog" );
+link( "catalog.new", "catalog" );
+unlink( "catalog.new" );


Property changes on: buildfarm/bin/rest-interface/getcatalog
___________________________________________________________________
Added: svn:executable
   + *

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.