SF.net SVN: gar:[23641] csw/mgar/gar/v2
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun May 18 13:17:57 CEST 2014
Revision: 23641
http://sourceforge.net/p/gar/code/23641
Author: wahwah
Date: 2014-05-18 11:17:56 +0000 (Sun, 18 May 2014)
Log Message:
-----------
uwatch: Use proxies
Uwatch used urllib2.urlopen, which ignores the *_proxy env variable family.
It's possible to use proxies with urllib2, but it's awkward.
http://stackoverflow.com/questions/1450132/proxy-with-urllib2
Instead, we can set HTTP_PROXY and use the 'requests' module which supports
them out of the box.
Modified Paths:
--------------
csw/mgar/gar/v2/gar.conf.mk
csw/mgar/gar/v2/gar.lib.mk
csw/mgar/gar/v2/lib/python/uwatch.py
Modified: csw/mgar/gar/v2/gar.conf.mk
===================================================================
--- csw/mgar/gar/v2/gar.conf.mk 2014-05-18 11:16:46 UTC (rev 23640)
+++ csw/mgar/gar/v2/gar.conf.mk 2014-05-18 11:17:56 UTC (rev 23641)
@@ -202,10 +202,11 @@
DEF_BASE_PKGS += CSWpy-cjson
DEF_BASE_PKGS += CSWpy-dateutil
DEF_BASE_PKGS += CSWpy-libmagic
+DEF_BASE_PKGS += CSWpy-mysql
DEF_BASE_PKGS += CSWpy-progressbar
DEF_BASE_PKGS += CSWpy-pyelftools
+DEF_BASE_PKGS += CSWpy-requests
DEF_BASE_PKGS += CSWpy-sqlobject
-DEF_BASE_PKGS += CSWpy-mysql
DEF_BASE_PKGS += CSWpython
DEF_BASE_PKGS += CSWwget
DEF_BASE_PKGS += CSWxz
Modified: csw/mgar/gar/v2/gar.lib.mk
===================================================================
--- csw/mgar/gar/v2/gar.lib.mk 2014-05-18 11:16:46 UTC (rev 23640)
+++ csw/mgar/gar/v2/gar.lib.mk 2014-05-18 11:17:56 UTC (rev 23641)
@@ -275,9 +275,9 @@
exit 1 ; \
fi ; \
if [ ! -n '$(UFILES_REGEX)' ]; then \
- VERSIONLIST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-version-list --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)"` ; \
+ VERSIONLIST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-version-list --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)"` ; \
else \
- VERSIONLIST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-version-list --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)"` ; \
+ VERSIONLIST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-version-list --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)"` ; \
fi ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch get-upstream-version-list. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
@@ -318,9 +318,9 @@
exit 1 ; \
fi ; \
if [ ! -n '$(UFILES_REGEX)' ]; then \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)"` ; \
else \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)"` ; \
fi ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch get-upstream-latest-version. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
@@ -357,9 +357,9 @@
exit 1 ; \
fi ; \
if [ ! -n '$(UFILES_REGEX)' ]; then \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)" --current-version="$(VERSION)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)" --current-version="$(VERSION)"` ; \
else \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --current-version="$(VERSION)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --current-version="$(VERSION)"` ; \
fi ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch check-upstream. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
@@ -397,9 +397,9 @@
exit 1 ; \
fi ; \
if [ ! -n '$(UFILES_REGEX)' ]; then \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)" --current-version="$(VERSION)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)" --current-version="$(VERSION)"` ; \
else \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --current-version="$(VERSION)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch check-upstream --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --current-version="$(VERSION)"` ; \
fi ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch check-upstream. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
@@ -410,7 +410,7 @@
if [ ! -d "../branches/upgrade_from_$(VERSION)_to_$$LATEST" ] ; then \
if [ -n "$$LATEST" ] ; then \
echo "$(NAME) : a new version of upstream files is available. Creating upgrade branch from version $(VERSION) to $$LATEST"; \
- VERSIONUPGRADE=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch upgrade-to-version --current-version="$(VERSION)" --target-version="$$LATEST"` ; \
+ VERSIONUPGRADE=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch upgrade-to-version --current-version="$(VERSION)" --target-version="$$LATEST"` ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch upgrade-to-version. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
echo "$$VERSIONUPGRADE" ; \
@@ -444,7 +444,7 @@
GARPATH=`echo $$line | awk '{ print $$1 }'` ; \
CATALOGNAME=`echo $$line | awk '{ print $$2 }'` ; \
PKGNAME=`echo $$line | awk '{ print $$3 }'` ; \
- REPORTVERSION=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Upstream Watch is disabled" --uwatch-deactivated` ; \
+ REPORTVERSION=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Upstream Watch is disabled" --uwatch-deactivated` ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch update-package-version-database --uwatch-deactivated. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
echo "$$REPORTVERSION" ; \
@@ -472,7 +472,7 @@
GARPATH=`echo $$line | awk '{ print $$1 }'` ; \
CATALOGNAME=`echo $$line | awk '{ print $$2 }'` ; \
PKGNAME=`echo $$line | awk '{ print $$3 }'` ; \
- REPORTVERSION=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --uwatch-error --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Upstream Watch configuration error" ` ; \
+ REPORTVERSION=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --uwatch-error --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Upstream Watch configuration error" ` ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch update-package-version-database --uwatch-error. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
echo "$$REPORTVERSION" ; \
@@ -482,9 +482,9 @@
exit 1 ; \
fi ; \
if [ ! -n '$(UFILES_REGEX)' ]; then \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --catalog-name="$(CATALOGNAME)"` ; \
else \
- LATEST=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)"` ; \
+ LATEST=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch get-upstream-latest-version --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)"` ; \
fi ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch get-upstream-latest-version. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
@@ -493,7 +493,7 @@
GARPATH=`echo $$line | awk '{ print $$1 }'` ; \
CATALOGNAME=`echo $$line | awk '{ print $$2 }'` ; \
PKGNAME=`echo $$line | awk '{ print $$3 }'` ; \
- REPORTVERSION=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --uwatch-error --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="$$LATEST" ` ; \
+ REPORTVERSION=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --uwatch-error --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="$$LATEST" ` ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch update-package-version-database --uwatch-error. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
echo "$$REPORTVERSION" ; \
@@ -507,14 +507,14 @@
CATALOGNAME=`echo $$line | awk '{ print $$2 }'` ; \
PKGNAME=`echo $$line | awk '{ print $$3 }'` ; \
if [ ! -n '$(UFILES_REGEX)' ]; then \
- REPORTVERSION=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version=$(VERSION) --upstream-version="$$LATEST" --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Successful" ` ; \
+ REPORTVERSION=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version=$(VERSION) --upstream-version="$$LATEST" --upstream-url="$(UPSTREAM_MASTER_SITES)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Successful" ` ; \
else \
- REPORTVERSION=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version=$(VERSION) --upstream-version="$$LATEST" --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Successful" ` ; \
+ REPORTVERSION=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version=$(VERSION) --upstream-version="$$LATEST" --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="Successful" ` ; \
fi ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch update-package-version-database. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
echo "$$REPORTVERSION" ; \
- REPORTVERSION=`http_proxy=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --uwatch-error --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="$$REPORTVERSION" ` ; \
+ REPORTVERSION=`HTTPS_PROXY=$(https_proxy) HTTP_PROXY=$(http_proxy) ftp_proxy=$(ftp_proxy) $(GARBIN)/uwatch update-package-version-database --catalog-name="$$CATALOGNAME" --package-name="$$PKGNAME" --execution-date="$$EXECUTIONDATE" --gar-path="$$GARPATH" --gar-version="$(VERSION)" --uwatch-error --upstream-url="$(UPSTREAM_MASTER_SITES)" --regexp="$(UFILES_REGEX)" --gar-distfiles="$(DISTFILES)" --uwatch-output="$$REPORTVERSION" ` ; \
if [ "$$?" -ne "0" ] ; then \
echo "Error occured while executing uwatch update-package-version-database --uwatch-error. Please check configuration with target get-uwatch-configuration. Here is the output of uwatch command :" ; \
echo "$$REPORTVERSION" ; \
Modified: csw/mgar/gar/v2/lib/python/uwatch.py
===================================================================
--- csw/mgar/gar/v2/lib/python/uwatch.py 2014-05-18 11:16:46 UTC (rev 23640)
+++ csw/mgar/gar/v2/lib/python/uwatch.py 2014-05-18 11:17:56 UTC (rev 23641)
@@ -36,7 +36,6 @@
# Import all the needed modules
import sys
-import string
import re
import os
import shutil
@@ -46,8 +45,8 @@
import datetime
import ConfigParser
import logging
+import requests
-from urllib2 import Request, urlopen, URLError
from optparse import OptionParser
# ---------------------------------------------------------------------------------------------------------------------
@@ -594,23 +593,18 @@
try:
# Request the upstream URL and open it
- req = Request(url)
- response = urlopen(req)
+ # req = Request(url)
+ # response = urlopen(req)
+ response = requests.get(url)
+ response.raise_for_status()
- except URLError, e:
- if hasattr(e, 'reason'):
- print 'We failed to reach a server during retrieval of : ' + url
- print 'Reason: ', e.reason
- elif hasattr(e, 'code'):
- print 'The server couldn\'t fulfill the request during retrieval of : ' + url
- print 'Error code: ', e.code
-
- # Check for response code value. We should get a 200
- raise UpstreamUrlRetrievalFailedException(url)
-
+ except (requests.exceptions.ConnectionError,
+ requests.exceptions.HTTPError) as e:
+ print e
+ raise UpstreamUrlRetrievalFailedException(' '.join((url, unicode(e))))
else:
# everything is fine, retrieve the page content
- return response.read()
+ return response.text
# -----------------------------------------------------------------------------------------------------------------
@@ -621,11 +615,11 @@
# an elements can be a string or a number
# at each step we extract the next elements of the two version strings and compare them
- if isinstance(version1, str) == False:
+ if isinstance(version1, basestring) == False:
print "Version is not a string. Please check environnement variable UFILES_REGEX"
print version1
- if isinstance(version2, str) == False:
+ if isinstance(version2, basestring) == False:
print "Version is not a string. Please check environnement variable UFILES_REGEX"
print version2
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list