[csw-devel] SF.net SVN: gar:[2639] csw/mgar/gar

wbonnet at users.sourceforge.net wbonnet at users.sourceforge.net
Thu Dec 18 00:39:14 CET 2008


Revision: 2639
          http://gar.svn.sourceforge.net/gar/?rev=2639&view=rev
Author:   wbonnet
Date:     2008-12-17 23:39:14 +0000 (Wed, 17 Dec 2008)

Log Message:
-----------
Add support for Sourceforge download page parsing

Modified Paths:
--------------
    csw/mgar/gar/v1/bin/upstream_watch
    csw/mgar/gar/v1/gar.conf.mk
    csw/mgar/gar/v1/gar.lib.mk
    csw/mgar/gar/v2/gar.lib.mk

Modified: csw/mgar/gar/v1/bin/upstream_watch
===================================================================
--- csw/mgar/gar/v1/bin/upstream_watch	2008-12-17 21:58:21 UTC (rev 2638)
+++ csw/mgar/gar/v1/bin/upstream_watch	2008-12-17 23:39:14 UTC (rev 2639)
@@ -108,11 +108,64 @@
 	return \@file_list;
 }
 
+sub get_file_list_wget_sourceforge
+{
+	my $url = shift;
+	my @file_list;
+
+	my $http_proxy_cmd = "";
+	my $wget_command ="wget";
+	my $filename = "";
+
+	# lftp doesn't seem to obey the _PROXY env variable
+	# we must manually set them
+	$http_proxy_cmd = "set http:proxy $ENV{HTTP_PROXY};" if exists $ENV{HTTP_PROXY};
+	$http_proxy_cmd = "set http:proxy $ENV{http_proxy};" if exists $ENV{http_proxy};
+	$wget_command   = "$http_proxy_cmd ; wget --proxy=on" if exists $ENV{http_proxy_cmd};
+
+	open (my $FH, "$wget_command -qO- $url 2>/dev/null | grep class | grep selected | grep li | grep Download | ");
+
+	if (my $line = <$FH>) {
+		my @cols = split (/"/, $line);
+		$filename = $cols[3];
+		chomp ($filename);
+	}
+	else {
+		close ($FH);
+		return \@file_list;
+	}
+
+	close ($FH);
+
+	$url = "http://downloads.sourceforge.net" . $filename;
+	open ($FH, "$wget_command -qO- $url 2>/dev/null | grep $filename | grep package_id= | grep release_id | ");
+
+	if (my $line = <$FH>) {
+		my @cols = split (/>/, $line);
+		my $filename = $cols[2];
+		chomp ($filename);
+		@cols = split (/</, $filename);
+		$filename = $cols[0];
+		chomp ($filename);
+		push (@file_list, $filename);
+	}
+	else {
+		close ($FH);
+		return \@file_list;
+	}
+
+	close ($FH);
+
+	return \@file_list;
+}
+
 my $help;
 my @urls;
+my $use_sf = 0;
 
 Getopt::Long::Configure ("no_ignore_case");  
 GetOptions("h|help" => \$help,
+	   "s|use_sf=i" => \$use_sf,
 	   "u|url=s@" => \@urls);
 
 if ($help) {
@@ -126,9 +179,14 @@
 
 	my $newest_version = 0;
 	my $newest_file = "";
+	my $file_list ;
 
 	foreach my $url (@urls) {
-		my $file_list = get_file_list_lftp ($url);
+		if ($use_sf != 0) {
+			$file_list = get_file_list_wget_sourceforge ($url);
+		} else {	
+			$file_list = get_file_list_lftp ($url);
+}
 
 		foreach my $file (@{$file_list}) {			
 			if ($file =~ /^$file_pattern$/) {

Modified: csw/mgar/gar/v1/gar.conf.mk
===================================================================
--- csw/mgar/gar/v1/gar.conf.mk	2008-12-17 21:58:21 UTC (rev 2638)
+++ csw/mgar/gar/v1/gar.conf.mk	2008-12-17 23:39:14 UTC (rev 2639)
@@ -233,6 +233,8 @@
 SF_MIRRORS  ?= http://downloads.sourceforge.net/$(SF_PROJ)/
 # Keep this for compatibility
 SF_MIRROR    = $(firstword $(SF_MIRRORS))
+SF_PROJECT_SHOWFILE ?= http://sourceforge.net/project/showfiles.php?group_id
+UPSTREAM_USE_SF 	?= 0
 
 # GNU
 GNU_SITE     = http://mirrors.kernel.org/

Modified: csw/mgar/gar/v1/gar.lib.mk
===================================================================
--- csw/mgar/gar/v1/gar.lib.mk	2008-12-17 21:58:21 UTC (rev 2638)
+++ csw/mgar/gar/v1/gar.lib.mk	2008-12-17 23:39:14 UTC (rev 2639)
@@ -113,8 +113,9 @@
 
 UPSTREAM_MASTER_SITES ?= $(MASTER_SITES)
 UW_ARGS = $(addprefix -u ,$(UPSTREAM_MASTER_SITES))
+SF_ARGS = $(addprefix -s ,$(UPSTREAM_USE_SF))
 ifneq ($(UFILES_REGEX), "")    
-	FILES2CHECK = $(shell http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/upstream_watch $(UW_ARGS) $(addsuffix ',$(addprefix ',$(UFILES_REGEX)))) 
+	FILES2CHECK = $(shell http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/upstream_watch $(UW_ARGS) $(SF_ARGS) $(addsuffix ',$(addprefix ',$(UFILES_REGEX)))) 
 else
 	FILES2CHECK = ""
 endif
@@ -130,7 +131,7 @@
 			else \
 				if echo $(DISTFILES) | grep -w $$FILE >/dev/null; then \
 					PACKAGE_UP_TO_DATE=1; \
-	                echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
+					echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
 				else \
 					NEW_FILES="$$FILE $$NEW_FILES"; \
 				fi; \
@@ -138,8 +139,8 @@
 			$(MAKE) checknew-$$FILE >/dev/null; \
 		done; \
 		if test -z "$$NEW_FILES" ; then \
-  			if [ ! -n '$(UFILES_REGEX)' ]; then \
-                echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
+			if [ ! -n '$(UFILES_REGEX)' ]; then \
+				echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
 #				{ echo ""; \
 #				  echo "Hello dear $(GARNAME) maintainer,"; \
 #				  echo ""; \
@@ -149,37 +150,37 @@
 #				  echo ""; \
 #				  echo "--"; \
 #				  echo "Kindest regards"; \
-#				  echo "The upstream notification job"; } | $(GARBIN)/mail2maintainer -s '[svn] $(GARNAME) upstream update notification' $(GARNAME); \
+#				  echo "upstream notification job"; } | $(GARBIN)/mail2maintainer -s '[svn] $(GARNAME) upstream update notification' $(GARNAME); \
 			else \
-	  			if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
+				if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
 					echo "$(GARNAME) : Warning no files to check ! $(FILES2CHECK)" ; \
 					echo "$(GARNAME) :     UPSTREAM_MASTER_SITES is $(UPSTREAM_MASTER_SITES)" ; \
 					echo "$(GARNAME) :     DISTNAME is $(DISTNAME)" ; \
 					echo "$(GARNAME) :     UFILES_REGEX is : $(UFILES_REGEX)" ; \
 					echo "$(GARNAME) : Please check configuration" ; \
-	    		fi; \
-    		fi; \
-        else \
+				fi; \
+			fi; \
+		else \
 			echo "$(GARNAME) : new upstream files available: $$NEW_FILES"; \
-			{ echo ""; \
-			  echo "Hello dear $(GARNAME) maintainer,"; \
-			  echo ""; \
-			  echo "The upstream notification job has detected the availability of new files for $(GARNAME)."; \
-			  echo ""; \
-			  echo "The following upstream file(s):"; \
-			  echo "    $$NEW_FILES"; \
-			  echo ""; \
-			  echo "is/are available at the following url(s):"; \
-			  echo "    $(UPSTREAM_MASTER_SITES)"; \
-			  echo ""; \
-			  echo "Please consider updating your package." ; \
-			  echo ""; \
-			  echo "--"; \
-			  echo "Kindest regards"; \
-			  echo "The upstream notification job"; } | $(GARBIN)/mail2maintainer -s '[svn] $(GARNAME) upstream update notification' $(GARNAME); \
-        fi; \
+			{	echo ""; \
+				echo "Hello dear $(GARNAME) maintainer,"; \
+				echo ""; \
+				echo "The upstream notification job has detected the availability of new files for $(GARNAME)."; \
+				echo ""; \
+				echo "The following upstream file(s):"; \
+				echo "    $$NEW_FILES"; \
+				echo ""; \
+				echo "is/are available at the following url(s):"; \
+				echo "    $(UPSTREAM_MASTER_SITES)"; \
+				echo ""; \
+				echo "Please consider updating your package." ; \
+				echo ""; \
+				echo "--"; \
+				echo "Kindest regards"; \
+				echo "upstream notification job"; } | $(GARBIN)/mail2maintainer -s '[svn] $(GARNAME) upstream update notification' $(GARNAME); \
+		fi; \
 	fi
-	
+		
 check-upstream: 
 	@if [ -n '$(FILES2CHECK)' ]; then \
 		NEW_FILES=""; \
@@ -191,7 +192,7 @@
 			else \
 				if echo $(DISTFILES) | grep -w $$FILE >/dev/null; then \
 					PACKAGE_UP_TO_DATE=1; \
-	                echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
+					echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
 				else \
 					NEW_FILES="$$FILE $$NEW_FILES"; \
 				fi; \
@@ -199,20 +200,20 @@
 			$(MAKE) checknew-$$FILE >/dev/null; \
 		done; \
 		if test -z "$$NEW_FILES" ; then \
-  			if [ ! -n '$(UFILES_REGEX)' ]; then \
-                echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
+			if [ ! -n '$(UFILES_REGEX)' ]; then \
+				echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
 			else \
-	  			if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
+				if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
 					echo "$(GARNAME) : Warning no files to check ! $(FILES2CHECK)" ; \
 					echo "$(GARNAME) :     UPSTREAM_MASTER_SITES is $(UPSTREAM_MASTER_SITES)" ; \
 					echo "$(GARNAME) :     DISTNAME is $(DISTNAME)" ; \
 					echo "$(GARNAME) :     UFILES_REGEX is : $(UFILES_REGEX)" ; \
 					echo "$(GARNAME) : Please check configuration" ; \
-	    		fi; \
-    		fi; \
-        else \
+				fi; \
+			fi; \
+		else \
 			echo "$(GARNAME) : new upstream files available: $$NEW_FILES"; \
-        fi; \
+		fi; \
 	fi
 	
 checknew-%:

Modified: csw/mgar/gar/v2/gar.lib.mk
===================================================================
--- csw/mgar/gar/v2/gar.lib.mk	2008-12-17 21:58:21 UTC (rev 2638)
+++ csw/mgar/gar/v2/gar.lib.mk	2008-12-17 23:39:14 UTC (rev 2639)
@@ -111,8 +111,9 @@
 
 UPSTREAM_MASTER_SITES ?= $(MASTER_SITES)
 UW_ARGS = $(addprefix -u ,$(UPSTREAM_MASTER_SITES))
+SF_ARGS = $(addprefix -s ,$(UPSTREAM_USE_SF))
 ifneq ($(UFILES_REGEX), "")    
-	FILES2CHECK = $(shell http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/upstream_watch $(UW_ARGS) $(addsuffix ',$(addprefix ',$(UFILES_REGEX)))) 
+	FILES2CHECK = $(shell http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/upstream_watch $(UW_ARGS) $(SF_ARGS) $(addsuffix ',$(addprefix ',$(UFILES_REGEX)))) 
 else
 	FILES2CHECK = ""
 endif
@@ -128,7 +129,7 @@
 			else \
 				if echo $(DISTFILES) | grep -w $$FILE >/dev/null; then \
 					PACKAGE_UP_TO_DATE=1; \
-	                echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
+					echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
 				else \
 					NEW_FILES="$$FILE $$NEW_FILES"; \
 				fi; \
@@ -136,8 +137,8 @@
 			$(MAKE) checknew-$$FILE >/dev/null; \
 		done; \
 		if test -z "$$NEW_FILES" ; then \
-  			if [ ! -n '$(UFILES_REGEX)' ]; then \
-                echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
+			if [ ! -n '$(UFILES_REGEX)' ]; then \
+				echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
 #				{ echo ""; \
 #				  echo "Hello dear $(GARNAME) maintainer,"; \
 #				  echo ""; \
@@ -149,33 +150,33 @@
 #				  echo "Kindest regards"; \
 #				  echo "upstream notification job"; } | $(GARBIN)/mail2maintainer -s '[svn] $(GARNAME) upstream update notification' $(GARNAME); \
 			else \
-	  			if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
+				if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
 					echo "$(GARNAME) : Warning no files to check ! $(FILES2CHECK)" ; \
 					echo "$(GARNAME) :     UPSTREAM_MASTER_SITES is $(UPSTREAM_MASTER_SITES)" ; \
 					echo "$(GARNAME) :     DISTNAME is $(DISTNAME)" ; \
 					echo "$(GARNAME) :     UFILES_REGEX is : $(UFILES_REGEX)" ; \
 					echo "$(GARNAME) : Please check configuration" ; \
-	    		fi; \
-    		fi; \
-        else \
+				fi; \
+			fi; \
+		else \
 			echo "$(GARNAME) : new upstream files available: $$NEW_FILES"; \
-			{ echo ""; \
-			  echo "Hello dear $(GARNAME) maintainer,"; \
-			  echo ""; \
-			  echo "The upstream notification job has detected the availability of new files for $(GARNAME)."; \
-			  echo ""; \
-			  echo "The following upstream file(s):"; \
-			  echo "    $$NEW_FILES"; \
-			  echo ""; \
-			  echo "is/are available at the following url(s):"; \
-			  echo "    $(UPSTREAM_MASTER_SITES)"; \
-			  echo ""; \
-			  echo "Please consider updating your package." ; \
-			  echo ""; \
-			  echo "--"; \
-			  echo "Kindest regards"; \
-			  echo "upstream notification job"; } | $(GARBIN)/mail2maintainer -s '[svn] $(GARNAME) upstream update notification' $(GARNAME); \
-        fi; \
+			{	echo ""; \
+				echo "Hello dear $(GARNAME) maintainer,"; \
+				echo ""; \
+				echo "The upstream notification job has detected the availability of new files for $(GARNAME)."; \
+				echo ""; \
+				echo "The following upstream file(s):"; \
+				echo "    $$NEW_FILES"; \
+				echo ""; \
+				echo "is/are available at the following url(s):"; \
+				echo "    $(UPSTREAM_MASTER_SITES)"; \
+				echo ""; \
+				echo "Please consider updating your package." ; \
+				echo ""; \
+				echo "--"; \
+				echo "Kindest regards"; \
+				echo "upstream notification job"; } | $(GARBIN)/mail2maintainer -s '[svn] $(GARNAME) upstream update notification' $(GARNAME); \
+		fi; \
 	fi
 		
 check-upstream: 
@@ -189,7 +190,7 @@
 			else \
 				if echo $(DISTFILES) | grep -w $$FILE >/dev/null; then \
 					PACKAGE_UP_TO_DATE=1; \
-	                echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
+					echo "$(GARNAME) : Package is up-to-date. Current version is $$FILE" ; \
 				else \
 					NEW_FILES="$$FILE $$NEW_FILES"; \
 				fi; \
@@ -197,20 +198,20 @@
 			$(MAKE) checknew-$$FILE >/dev/null; \
 		done; \
 		if test -z "$$NEW_FILES" ; then \
-  			if [ ! -n '$(UFILES_REGEX)' ]; then \
-                echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
+			if [ ! -n '$(UFILES_REGEX)' ]; then \
+				echo "$(GARNAME) : Warning UFILES_REGEX is not set : $(UFILES_REGEX)" ; \
 			else \
-	  			if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
+				if [ "$$PACKAGE_UP_TO_DATE" -eq "0" ]; then \
 					echo "$(GARNAME) : Warning no files to check ! $(FILES2CHECK)" ; \
 					echo "$(GARNAME) :     UPSTREAM_MASTER_SITES is $(UPSTREAM_MASTER_SITES)" ; \
 					echo "$(GARNAME) :     DISTNAME is $(DISTNAME)" ; \
 					echo "$(GARNAME) :     UFILES_REGEX is : $(UFILES_REGEX)" ; \
 					echo "$(GARNAME) : Please check configuration" ; \
-	    		fi; \
-    		fi; \
-        else \
+				fi; \
+			fi; \
+		else \
 			echo "$(GARNAME) : new upstream files available: $$NEW_FILES"; \
-        fi; \
+		fi; \
 	fi
 	
 checknew-%:


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



More information about the devel mailing list