[csw-devel] SF.net SVN: gar:[19675] csw/mgar/gar/v2/lib/python/integrate_catalogs.py

wahwah at users.sourceforge.net wahwah at users.sourceforge.net
Mon Nov 12 16:57:10 CET 2012


Revision: 19675
          http://gar.svn.sourceforge.net/gar/?rev=19675&view=rev
Author:   wahwah
Date:     2012-11-12 15:57:09 +0000 (Mon, 12 Nov 2012)
Log Message:
-----------
integrate_catalogs: Use curl instead of pkgdb

It's way faster. Currently, most of the time is spent on the server side.
Package insertios are significantly slower than deletions.

Modified Paths:
--------------
    csw/mgar/gar/v2/lib/python/integrate_catalogs.py

Modified: csw/mgar/gar/v2/lib/python/integrate_catalogs.py
===================================================================
--- csw/mgar/gar/v2/lib/python/integrate_catalogs.py	2012-11-12 15:55:20 UTC (rev 19674)
+++ csw/mgar/gar/v2/lib/python/integrate_catalogs.py	2012-11-12 15:57:09 UTC (rev 19675)
@@ -23,28 +23,47 @@
 import rest
 import sys
 import urllib2
+import re
 
 
 CATALOG_MOD_TMPL = """#!/bin/bash
 # Catalog modification (not integration yet): $catrel_from -> $catrel_to
 # Generated by $prog
 
+
+if ! grep buildfarm ~/.netrc
+then
+  touch ~/.netrc
+  chmod 0600 ~/.netrc
+  echo >> ~/.netrc \
+    "machine buildfarm.opencsw.org login \${LOGNAME} password \$(cat /etc/opt/csw/releases/auth/\${LOGNAME})"
+fi
+
 set -x
 
-PKGDB=bin/pkgdb
+readonly CURL="curl --netrc"
+readonly REST_URL=http://buildfarm.opencsw.org/releases/
 
+function _add_to_cat {
+  \${CURL} -X PUT \${REST_URL}catalogs/$1/$2/$3/$4/
+}
+
+function _del_from_cat {
+  \${CURL} -X DELETE \${REST_URL}catalogs/$1/$2/$3/$4/
+}
+
 #for catalogname in $sorted($diffs_by_catalogname):
 #if "new_pkgs" in $diffs_by_catalogname[$catalogname]:
 function new_pkg_$catalogname {
 #for arch, osrel, new_pkg in $diffs_by_catalogname[$catalogname]["new_pkgs"]:
   # adding $new_pkg["basename"]
-  \${PKGDB} add-to-cat $osrel $arch $catrel_to $new_pkg["md5_sum"]
+  _add_to_cat $catrel_to $arch $osrel $new_pkg["md5_sum"]
 #end for
 }
 function undo_new_pkg_$catalogname {
 #for arch, osrel, new_pkg in $diffs_by_catalogname[$catalogname]["new_pkgs"]:
-  # adding $new_pkg["basename"]
-  \${PKGDB} del-from-cat $osrel $arch $catrel_to $new_pkg["md5_sum"]
+  # UNDO adding $new_pkg["basename"]
+  _del_from_cat $catrel_to $arch $osrel $new_pkg["md5_sum"]
 #end for
 }
 #end if
@@ -52,13 +71,13 @@
 function remove_pkg_$catalogname {
 #for arch, osrel, rem_pkg in $diffs_by_catalogname[$catalogname]["removed_pkgs"]:
   # removing $rem_pkg["basename"]
-  \${PKGDB} del-from-cat $osrel $arch $catrel_to $rem_pkg["md5_sum"]
+  _del_from_cat $catrel_to $arch $osrel $rem_pkg["md5_sum"]
 #end for
 }
 function undo_remove_pkg_$catalogname {
 #for arch, osrel, rem_pkg in $diffs_by_catalogname[$catalogname]["removed_pkgs"]:
-  # removing $rem_pkg["basename"]
-  \${PKGDB} add-to-cat $osrel $arch $catrel_to $rem_pkg["md5_sum"]
+  # UNDO removing $rem_pkg["basename"]
+  _add_to_cat $catrel_to $arch $osrel $rem_pkg["md5_sum"]
 #end for
 }
 #end if
@@ -75,15 +94,15 @@
   # WARNING: DOWNGRADE
 #end if
   # $catalogname $up_pkg_pair["direction"] from $up_pkg_pair["from"]["version"] to $up_pkg_pair["to"]["version"]
-  \${PKGDB} del-from-cat $osrel $arch $catrel_to $up_pkg_pair["from"]["md5_sum"]
-  \${PKGDB} add-to-cat $osrel $arch $catrel_to $up_pkg_pair["to"]["md5_sum"]
+  _del_from_cat $catrel_to $arch $osrel $up_pkg_pair["from"]["md5_sum"]
+  _add_to_cat $catrel_to $arch $osrel $up_pkg_pair["to"]["md5_sum"]
 #end for
 }
 function undo_upgrade_$catalogname {
 #for arch, osrel, up_pkg_pair in $diffs_by_catalogname[$catalogname]["updated_pkgs"]:
   # UNDO of $catalogname $up_pkg_pair["direction"] from $up_pkg_pair["from"]["version"] to $up_pkg_pair["to"]["version"]
-  \${PKGDB} del-from-cat $osrel $arch $catrel_to $up_pkg_pair["to"]["md5_sum"]
-  \${PKGDB} add-to-cat $osrel $arch $catrel_to $up_pkg_pair["from"]["md5_sum"]
+  _del_from_cat $catrel_to $arch $osrel $up_pkg_pair["to"]["md5_sum"]
+  _add_to_cat $catrel_to $arch $osrel $up_pkg_pair["from"]["md5_sum"]
 #end for
 }
 #end if
@@ -116,7 +135,6 @@
 #end for
 """
 
-
 class Error(Exception):
   """Generic error."""
 
@@ -202,10 +220,12 @@
       # By passing the catalogs (as arguments) in reverse order, we get
       # packages to be updated in new_pkgs, and so forth.
       for pkg in new_pkgs:
-        catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+        catalogname_d = diffs_by_catalogname.setdefault(
+            (pkg["catalogname"]), {})
         catalogname_d.setdefault("new_pkgs", []).append((arch, osrel, pkg))
       for pkg in removed_pkgs:
-        catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+        catalogname_d = diffs_by_catalogname.setdefault(
+            (pkg["catalogname"]), {})
         catalogname_d.setdefault("removed_pkgs", []).append((arch, osrel, pkg))
       for pkg_pair in updated_pkgs:
         update_decision_by_type = {
@@ -215,7 +235,8 @@
         if (update_decision_by_type[pkg_pair["type"]]
             and (pkg_pair["direction"] == "upgrade" or include_downgrades)):
           pkg = pkg_pair["from"]
-          catalogname_d = diffs_by_catalogname.setdefault(pkg["catalogname"], {})
+          catalogname_d = diffs_by_catalogname.setdefault(
+              (pkg["catalogname"]), {})
           catalogname_d.setdefault("updated_pkgs", []).append((arch, osrel, pkg_pair))
   return diffs_by_catalogname
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



More information about the devel mailing list